fix(images): outages are 502, purges are not undone, bad pictures are not outages
- A timeout inside the provider (the outbound client's own deadline, which wraps context.DeadlineExceeded) was mistaken for the caller leaving and came out as a 500, so an older stored picture was not served. The decision now uses the caller's own context, and a build that runs out of its two minutes is an outage (502) too. - A purge of the Commons data could be undone by a read that was already running: the row was deleted, so the generation guard started from zero, and the files were written after the files were removed. Pictures are now kept only under a purge counter; BeginArtistPurge is called before the purge and a read that started earlier starts again, or is refused if the provider is off by then. - A redirect or an oversized picture from the picture host is a fault of that picture (no picture), not an outage, and the pictures are downloaded through a client of their own so that one bad file cannot open the breaker of the API calls. - The author drops the invisible characters that reorder or hide text. Artist pictures are decoded only up to 4 million pixels. - Tests: an outage that ends in a deadline, a build that times out, a purge during a read, a redirect and a huge picture from the host, a separate download client.
This commit is contained in:
parent
e752d15536
commit
9e85d4d1ea
10 changed files with 308 additions and 58 deletions
|
|
@ -59,6 +59,7 @@ const (
|
|||
// commonsMaxBody leaves room for a 960 pixel copy of a picture, which is some hundred kilobytes.
|
||||
const (
|
||||
commonsMinInterval = 300 * time.Millisecond
|
||||
commonsAPIMaxBody = 1 << 20
|
||||
commonsMaxBody = 8 << 20
|
||||
)
|
||||
|
||||
|
|
@ -159,11 +160,17 @@ func run() error {
|
|||
return wikipediaClient.Probe(ctx)
|
||||
})
|
||||
// Pictures of artists from Wikimedia Commons, through Wikidata, under free licences only.
|
||||
// The pictures are downloaded through a client of their own: they are far bigger than an
|
||||
// answer, and a picture host that misbehaves must not open the breaker of the API.
|
||||
commonsClient := commons.New(outbound.New(outbound.Options{
|
||||
UserAgentFunc: providerSettings.UserAgentFor(providers.Commons, version),
|
||||
MinInterval: commonsMinInterval,
|
||||
MaxBody: commonsAPIMaxBody,
|
||||
}), commons.WikidataURL, commons.CommonsURL).WithDownloader(outbound.New(outbound.Options{
|
||||
UserAgentFunc: providerSettings.UserAgentFor(providers.Commons, version),
|
||||
MinInterval: commonsMinInterval,
|
||||
MaxBody: commonsMaxBody,
|
||||
}), commons.WikidataURL, commons.CommonsURL)
|
||||
}))
|
||||
providerSettings.SetChecker(providers.Commons, func(ctx context.Context, _ string) error {
|
||||
return commonsClient.Probe(ctx)
|
||||
})
|
||||
|
|
@ -177,6 +184,9 @@ func run() error {
|
|||
Artists: discoveryRepo, Settings: providerSettings, MusicBrainz: musicbrainzClient, Commons: commonsClient,
|
||||
})
|
||||
providerSettings.SetPurger(func(ctx context.Context, name string) (providers.Purged, error) {
|
||||
if name == providers.Commons {
|
||||
pictures.BeginArtistPurge() // a picture being read right now must not be kept
|
||||
}
|
||||
out, err := discoveryRepo.Purge(ctx, name)
|
||||
if err != nil || name != providers.Commons {
|
||||
return out, err
|
||||
|
|
|
|||
|
|
@ -1088,7 +1088,8 @@ paths:
|
|||
Whose the picture is and under which licence, to be shown next to it (the licences ask for
|
||||
it). The author is plain text and may be empty; `license_url` may be empty when the licence
|
||||
has no address; `page_url` is the page of the file on Commons, where everything is.
|
||||
It answers like `GET /artists/{id}/image` (same errors), because the credit exists only with a picture.
|
||||
It answers like `GET /artists/{id}/image` (same errors), because the credit exists only with a
|
||||
picture: the first request for an artist can take as long as the picture does (several seconds).
|
||||
security: [{bearerAuth: []}]
|
||||
responses:
|
||||
'200':
|
||||
|
|
|
|||
|
|
@ -70,13 +70,33 @@ func (s *Service) ArtistImage(ctx context.Context, artistID int64, size Size) (*
|
|||
return s.open(kindArtist, artistID, size, built)
|
||||
}
|
||||
|
||||
// MaxArtistPixels is the largest copy of an artist's picture that is decoded. The copy
|
||||
// asked from Commons is 960 pixels wide, so this leaves room for a very tall one.
|
||||
const MaxArtistPixels = 4_000_000
|
||||
|
||||
// BeginArtistPurge must be called before the pictures of artists are taken back (their
|
||||
// records forgotten and their files removed). A read that started before it is not kept:
|
||||
// it starts again, and is refused if the provider is by then off.
|
||||
func (s *Service) BeginArtistPurge() {
|
||||
s.purgeMu.Lock()
|
||||
s.purges++
|
||||
s.purgeMu.Unlock()
|
||||
}
|
||||
|
||||
func (s *Service) purgeCount() uint64 {
|
||||
s.purgeMu.RLock()
|
||||
defer s.purgeMu.RUnlock()
|
||||
return s.purges
|
||||
}
|
||||
|
||||
// buildArtist asks the source, stores the picture and records the outcome. If the
|
||||
// record was cleared while it worked, what it found may be out of date: it starts again.
|
||||
// record was cleared or the pictures were taken back while it worked, what it found may
|
||||
// be out of date: it starts again.
|
||||
func (s *Service) buildArtist(artistID int64) (Record, error) {
|
||||
if s.artists == nil {
|
||||
return Record{}, ErrProviderDisabled
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), buildTimeout)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), s.buildTimeout)
|
||||
defer cancel()
|
||||
var rec Record
|
||||
for attempt := 0; attempt < 3; attempt++ {
|
||||
|
|
@ -84,14 +104,27 @@ func (s *Service) buildArtist(artistID int64) (Record, error) {
|
|||
if err != nil {
|
||||
return Record{}, fmt.Errorf("read picture record: %w", err)
|
||||
}
|
||||
rec, err = s.searchArtist(ctx, artistID)
|
||||
epoch := s.purgeCount()
|
||||
var pic *Processed
|
||||
rec, pic, err = s.searchArtist(ctx, artistID)
|
||||
if errors.Is(err, errPurged) {
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
// Nobody calls this context: when it ends it is the build that ran out of time,
|
||||
// which is an outage of the provider and not something to hide as an internal error.
|
||||
if ctx.Err() != nil && (errors.Is(err, context.DeadlineExceeded) || errors.Is(err, context.Canceled)) {
|
||||
err = fmt.Errorf("%w: %w", ErrSourceUnavailable, err)
|
||||
}
|
||||
return Record{}, err
|
||||
}
|
||||
rec.Gen = cur.Gen
|
||||
saved, err := s.store.Save(ctx, kindArtist, artistID, rec)
|
||||
saved, err := s.saveArtist(ctx, artistID, rec, pic, epoch)
|
||||
if errors.Is(err, errPurged) {
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return Record{}, fmt.Errorf("save picture record: %w", err)
|
||||
return Record{}, err
|
||||
}
|
||||
if saved {
|
||||
return rec, nil
|
||||
|
|
@ -100,27 +133,30 @@ func (s *Service) buildArtist(artistID int64) (Record, error) {
|
|||
return rec, nil
|
||||
}
|
||||
|
||||
func (s *Service) searchArtist(ctx context.Context, artistID int64) (Record, error) {
|
||||
// errPurged says the pictures were taken back since the build began.
|
||||
var errPurged = errors.New("the pictures were taken back while this was being built")
|
||||
|
||||
// searchArtist asks the source and, for a picture, prepares it (nil when there is none).
|
||||
// The files are written by saveArtist, together with the record, so that a purge cannot
|
||||
// fall between them.
|
||||
func (s *Service) searchArtist(ctx context.Context, artistID int64) (Record, *Processed, error) {
|
||||
select {
|
||||
case s.artistSl <- struct{}{}:
|
||||
defer func() { <-s.artistSl }()
|
||||
case <-ctx.Done():
|
||||
return Record{}, ctx.Err()
|
||||
return Record{}, nil, ctx.Err()
|
||||
}
|
||||
none := Record{Status: StatusNone, CheckedAt: s.now()}
|
||||
found, err := s.artists.ArtistImage(ctx, artistID)
|
||||
switch {
|
||||
case errors.Is(err, ErrNoImage):
|
||||
return none, nil
|
||||
return none, nil, nil
|
||||
case err != nil:
|
||||
return Record{}, err
|
||||
return Record{}, nil, err
|
||||
}
|
||||
p, err := Process(found.Data)
|
||||
p, err := ProcessLimited(found.Data, MaxArtistPixels)
|
||||
if err != nil {
|
||||
return none, nil // what the provider sent is not a picture: there is none to show
|
||||
}
|
||||
if err := s.write(kindArtist, artistID, p); err != nil {
|
||||
return Record{}, err
|
||||
return none, nil, nil // what the provider sent is not a usable picture: there is none to show
|
||||
}
|
||||
// The credit is part of the identity: the same picture under another author or licence
|
||||
// must not be taken for the same one by a client that kept the old credit.
|
||||
|
|
@ -130,7 +166,27 @@ func (s *Service) searchArtist(ctx context.Context, artistID int64) (Record, err
|
|||
return Record{
|
||||
Status: StatusOK, Source: SourceCommons, Credit: found.Credit, ETag: fmt.Sprintf("%x", h.Sum(nil)[:8]),
|
||||
Width: p.Width, Height: p.Height, CheckedAt: none.CheckedAt,
|
||||
}, nil
|
||||
}, &p, nil
|
||||
}
|
||||
|
||||
// saveArtist keeps what was found: the files and the record, unless the pictures were
|
||||
// taken back since the build began (errPurged) or a change moved the record's generation.
|
||||
func (s *Service) saveArtist(ctx context.Context, artistID int64, rec Record, pic *Processed, epoch uint64) (bool, error) {
|
||||
s.purgeMu.RLock()
|
||||
defer s.purgeMu.RUnlock()
|
||||
if s.purges != epoch {
|
||||
return false, errPurged
|
||||
}
|
||||
if pic != nil {
|
||||
if err := s.write(kindArtist, artistID, *pic); err != nil {
|
||||
return false, err
|
||||
}
|
||||
}
|
||||
saved, err := s.store.Save(ctx, kindArtist, artistID, rec)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("save picture record: %w", err)
|
||||
}
|
||||
return saved, nil
|
||||
}
|
||||
|
||||
// RemoveArtistFiles deletes the stored pictures of artists, and only those, and says how
|
||||
|
|
|
|||
|
|
@ -21,15 +21,24 @@ type fakeArtists struct {
|
|||
err error
|
||||
calls int32
|
||||
gate chan struct{}
|
||||
gate2 bool // the gate is never opened for this call: it waits for the context instead
|
||||
}
|
||||
|
||||
func (f *fakeArtists) ArtistImage(_ context.Context, id int64) (Found, error) {
|
||||
func (f *fakeArtists) ArtistImage(ctx context.Context, id int64) (Found, error) {
|
||||
atomic.AddInt32(&f.calls, 1)
|
||||
f.mu.Lock()
|
||||
gate, found, err := f.gate, f.found, f.err
|
||||
f.mu.Unlock()
|
||||
if gate != nil {
|
||||
<-gate
|
||||
if f.gate2 {
|
||||
select {
|
||||
case <-gate:
|
||||
case <-ctx.Done():
|
||||
return Found{}, ctx.Err()
|
||||
}
|
||||
} else {
|
||||
<-gate
|
||||
}
|
||||
}
|
||||
return found, err
|
||||
}
|
||||
|
|
@ -205,18 +214,77 @@ func TestManyAsksForAnArtistPictureAskTheProviderOnce(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
func TestAChangeDuringTheReadOfAnArtistPictureIsNotUndoneByIt(t *testing.T) {
|
||||
func TestAPurgeDuringTheReadIsNotUndoneByIt(t *testing.T) {
|
||||
r := newArtistRig(t)
|
||||
r.src.gate = make(chan struct{})
|
||||
done := make(chan error, 1)
|
||||
go func() { _, err := r.svc.ArtistImage(context.Background(), 7, Large); done <- err }()
|
||||
time.Sleep(50 * time.Millisecond) // the read has the picture and is about to keep it
|
||||
r.svc.BeginArtistPurge() // the admin takes the provider's data back...
|
||||
r.src.set(Found{}, ErrProviderDisabled) // ...and switches it off
|
||||
close(r.src.gate)
|
||||
if err := <-done; !errors.Is(err, ErrProviderDisabled) {
|
||||
t.Fatalf("err = %v, want the read to be refused once the provider is off", err)
|
||||
}
|
||||
if len(r.store.recs) != 0 {
|
||||
t.Errorf("records = %v: what the read found before the purge came back", r.store.recs)
|
||||
}
|
||||
if entries, _ := os.ReadDir(filepath.Join(r.data, "images", "artist")); len(entries) != 0 {
|
||||
t.Errorf("%d files left by a read that started before the purge", len(entries))
|
||||
}
|
||||
}
|
||||
|
||||
func TestAReadThatStartedBeforeAPurgeStartsAgainIfTheProviderIsStillOn(t *testing.T) {
|
||||
r := newArtistRig(t)
|
||||
r.src.gate = make(chan struct{})
|
||||
done := make(chan error, 1)
|
||||
go func() { _, err := r.svc.ArtistImage(context.Background(), 7, Large); done <- err }()
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
r.store.clear("artist", 7) // for example the purge of the provider
|
||||
r.svc.BeginArtistPurge()
|
||||
close(r.src.gate)
|
||||
<-done
|
||||
// What the read that started before the change found was not kept as the answer.
|
||||
if rec := r.store.recs["artist/7"]; rec.Status == StatusOK && rec.Gen == 0 {
|
||||
t.Errorf("record = %+v: an out of date build was saved over the change", rec)
|
||||
if err := <-done; err != nil {
|
||||
t.Fatalf("err = %v", err)
|
||||
}
|
||||
if n := atomic.LoadInt32(&r.src.calls); n != 2 {
|
||||
t.Errorf("the provider was asked %d times, want 2: the first answer was from before the purge", n)
|
||||
}
|
||||
if rec := r.store.recs["artist/7"]; rec.Status != StatusOK {
|
||||
t.Errorf("record = %+v", rec)
|
||||
}
|
||||
}
|
||||
|
||||
func TestABuildThatTakesTooLongIsAnOutageNotAnInternalError(t *testing.T) {
|
||||
r := newArtistRig(t)
|
||||
_, old := r.get(t, 7, Large) // a picture is stored
|
||||
r.now = r.now.Add(ArtistTTL + time.Hour)
|
||||
r.svc.buildTimeout = 40 * time.Millisecond
|
||||
r.src.gate = make(chan struct{}) // never opened during the build
|
||||
r.src.gate2 = true
|
||||
c, again := r.get(t, 7, Large)
|
||||
if !bytes.Equal(old, again) || c.Credit != credit {
|
||||
t.Error("the old picture was not served when the build timed out")
|
||||
}
|
||||
// With nothing stored the caller is told it is an outage (502), not a failure of ours (500).
|
||||
r2 := newArtistRig(t)
|
||||
r2.svc.buildTimeout = 40 * time.Millisecond
|
||||
r2.src.gate = make(chan struct{})
|
||||
r2.src.gate2 = true
|
||||
if _, err := r2.svc.ArtistImage(context.Background(), 8, Large); !errors.Is(err, ErrSourceUnavailable) {
|
||||
t.Errorf("err = %v, want ErrSourceUnavailable", err)
|
||||
}
|
||||
close(r.src.gate)
|
||||
close(r2.src.gate)
|
||||
}
|
||||
|
||||
func TestAHugePictureOfAnArtistIsRefused(t *testing.T) {
|
||||
r := newArtistRig(t)
|
||||
r.src.set(Found{Data: jpegBytes(t, 2500, 2500, color.White), Credit: credit}, nil) // 6 million pixels
|
||||
if _, err := r.svc.ArtistImage(context.Background(), 7, Large); !errors.Is(err, ErrNoImage) {
|
||||
t.Errorf("err = %v, want ErrNoImage: a copy of 960 pixels is never that big", err)
|
||||
}
|
||||
// The same picture is fine as the cover of an album (the limit there is larger).
|
||||
if _, err := Process(r.src.found.Data); err != nil {
|
||||
t.Errorf("Process: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -58,11 +58,13 @@ func (c *CommonsSource) active(ctx context.Context) (bool, error) {
|
|||
return true, nil
|
||||
}
|
||||
|
||||
// failed turns what went wrong into ErrSourceUnavailable, keeping the cause. A request
|
||||
// that was cancelled is not a failure of the provider and is returned as it is.
|
||||
func failed(err error) error {
|
||||
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
|
||||
return err
|
||||
// failed turns what went wrong into ErrSourceUnavailable, keeping the cause. When the
|
||||
// caller's own context has ended (it left, or ran out of time) that is returned instead:
|
||||
// it is not the provider's fault. A deadline buried in the error is not a sign of that, since
|
||||
// the outbound client's own timeout produces one when the provider is merely slow.
|
||||
func failed(ctx context.Context, err error) error {
|
||||
if ctxErr := ctx.Err(); ctxErr != nil {
|
||||
return ctxErr
|
||||
}
|
||||
return fmt.Errorf("%w: %w", ErrSourceUnavailable, err)
|
||||
}
|
||||
|
|
@ -90,7 +92,7 @@ func (c *CommonsSource) ArtistImage(ctx context.Context, artistID int64) (Found,
|
|||
case errors.Is(err, musicbrainz.ErrNotFound), errors.Is(err, musicbrainz.ErrBadRequest):
|
||||
return Found{}, ErrNoImage
|
||||
case err != nil:
|
||||
return Found{}, failed(err)
|
||||
return Found{}, failed(ctx, err)
|
||||
case item == "":
|
||||
return Found{}, ErrNoImage
|
||||
}
|
||||
|
|
@ -99,7 +101,7 @@ func (c *CommonsSource) ArtistImage(ctx context.Context, artistID int64) (Found,
|
|||
case errors.Is(err, commons.ErrNotFound), errors.Is(err, commons.ErrBadRequest):
|
||||
return Found{}, ErrNoImage
|
||||
case err != nil:
|
||||
return Found{}, failed(err)
|
||||
return Found{}, failed(ctx, err)
|
||||
case name == "":
|
||||
return Found{}, ErrNoImage
|
||||
}
|
||||
|
|
@ -109,7 +111,7 @@ func (c *CommonsSource) ArtistImage(ctx context.Context, artistID int64) (Found,
|
|||
errors.Is(err, commons.ErrUnusable), errors.Is(err, commons.ErrBadRequest):
|
||||
return Found{}, ErrNoImage
|
||||
case err != nil:
|
||||
return Found{}, failed(err)
|
||||
return Found{}, failed(ctx, err)
|
||||
}
|
||||
// The admin may have switched the provider off while the chain was being followed:
|
||||
// do not download, and do not hand over, what they no longer allow.
|
||||
|
|
@ -120,10 +122,10 @@ func (c *CommonsSource) ArtistImage(ctx context.Context, artistID int64) (Found,
|
|||
}
|
||||
data, err := c.Commons.Download(ctx, file.ThumbURL)
|
||||
switch {
|
||||
case errors.Is(err, commons.ErrBadRequest), errors.Is(err, commons.ErrNotFound):
|
||||
case errors.Is(err, commons.ErrBadRequest), errors.Is(err, commons.ErrNotFound), errors.Is(err, commons.ErrUnusable):
|
||||
return Found{}, ErrNoImage
|
||||
case err != nil:
|
||||
return Found{}, failed(err)
|
||||
return Found{}, failed(ctx, err)
|
||||
}
|
||||
if on, err := c.active(ctx); err != nil {
|
||||
return Found{}, err
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ package images
|
|||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
|
|
@ -194,10 +195,38 @@ func TestSwitchedOffWhileItWasWorkingNothingIsReturned(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
func TestACancelledRequestIsNotAFailureOfTheProvider(t *testing.T) {
|
||||
func TestAnOutageThatEndsInADeadlineIsStillAnOutage(t *testing.T) {
|
||||
// The outbound client's own timeout wraps context.DeadlineExceeded inside ErrUnavailable. It
|
||||
// is a failure of the provider (so an older picture can be served), not a cancellation.
|
||||
slow := fmt.Errorf("%w: %w", commons.ErrUnavailable, context.DeadlineExceeded)
|
||||
for name, tt := range map[string]func(*chain){
|
||||
"musicbrainz": func(c *chain) { c.mb.err = fmt.Errorf("%w: %w", musicbrainz.ErrUnavailable, context.DeadlineExceeded) },
|
||||
"wikidata": func(c *chain) { c.cm.nameErr = slow },
|
||||
"commons": func(c *chain) { c.cm.infoErr = slow },
|
||||
"the download": func(c *chain) { c.cm.loadErr = slow },
|
||||
} {
|
||||
c := newChain()
|
||||
tt(c)
|
||||
if _, err := c.src.ArtistImage(context.Background(), 1); !errors.Is(err, ErrSourceUnavailable) {
|
||||
t.Errorf("%s: err = %v, want ErrSourceUnavailable", name, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestARequestThatTheCallerCancelledIsReturnedAsCancelled(t *testing.T) {
|
||||
c := newChain()
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
c.mb.hook = cancel // the caller leaves while MusicBrainz is being asked
|
||||
c.mb.err = context.Canceled
|
||||
if _, err := c.src.ArtistImage(context.Background(), 1); !errors.Is(err, context.Canceled) {
|
||||
if _, err := c.src.ArtistImage(ctx, 1); !errors.Is(err, context.Canceled) || errors.Is(err, ErrSourceUnavailable) {
|
||||
t.Errorf("err = %v, want the cancellation itself", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAPictureThatCannotBeUsedFromTheHostIsNoPicture(t *testing.T) {
|
||||
c := newChain()
|
||||
c.cm.loadErr = commons.ErrUnusable // a redirect or a picture over the limit
|
||||
if _, err := c.src.ArtistImage(context.Background(), 1); !errors.Is(err, ErrNoImage) {
|
||||
t.Errorf("err = %v, want ErrNoImage", err)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -45,7 +45,11 @@ type Processed struct {
|
|||
// Process decodes a GIF, JPEG or PNG and returns it scaled down to fit LargeEdge and
|
||||
// SmallEdge (never enlarged), on a white background where it was transparent. The
|
||||
// same input always gives the same bytes.
|
||||
func Process(data []byte) (Processed, error) {
|
||||
func Process(data []byte) (Processed, error) { return ProcessLimited(data, MaxPixels) }
|
||||
|
||||
// ProcessLimited is Process with a smaller limit on the number of pixels, for pictures that
|
||||
// are known to be small (a 960 pixel copy is never millions of pixels high).
|
||||
func ProcessLimited(data []byte, maxPixels int64) (Processed, error) {
|
||||
if len(data) == 0 || len(data) > MaxSourceBytes {
|
||||
return Processed{}, fmt.Errorf("%w: %d bytes", ErrUnusable, len(data))
|
||||
}
|
||||
|
|
@ -53,7 +57,7 @@ func Process(data []byte) (Processed, error) {
|
|||
if err != nil {
|
||||
return Processed{}, fmt.Errorf("%w: %v", ErrUnusable, err)
|
||||
}
|
||||
if cfg.Width <= 0 || cfg.Height <= 0 || int64(cfg.Width)*int64(cfg.Height) > MaxPixels { // int64: a 32-bit int would overflow
|
||||
if cfg.Width <= 0 || cfg.Height <= 0 || int64(cfg.Width)*int64(cfg.Height) > maxPixels { // int64: a 32-bit int would overflow
|
||||
return Processed{}, fmt.Errorf("%w: %dx%d pixels", ErrUnusable, cfg.Width, cfg.Height)
|
||||
}
|
||||
src, _, err := image.Decode(bytes.NewReader(data))
|
||||
|
|
|
|||
|
|
@ -157,22 +157,31 @@ type Cover struct {
|
|||
// Service builds covers from the user's own files, on the first request, and keeps
|
||||
// them in the data folder.
|
||||
type Service struct {
|
||||
dir string
|
||||
store Store
|
||||
src Source
|
||||
embedded func(path string) ([]byte, error)
|
||||
now func() time.Time
|
||||
flight singleflight.Group
|
||||
slots chan struct{} // one per cover build allowed at the same time
|
||||
artistSl chan struct{} // the same for artist pictures, which wait for the network
|
||||
artists ArtistSource
|
||||
tidied sync.Map // the folders whose old temporary files were already removed
|
||||
dir string
|
||||
store Store
|
||||
src Source
|
||||
embedded func(path string) ([]byte, error)
|
||||
now func() time.Time
|
||||
flight singleflight.Group
|
||||
slots chan struct{} // one per cover build allowed at the same time
|
||||
artistSl chan struct{} // the same for artist pictures, which wait for the network
|
||||
artists ArtistSource
|
||||
buildTimeout time.Duration // how long one build may take
|
||||
|
||||
// purges counts the times the provider's pictures were taken back (BeginArtistPurge). A
|
||||
// build that started before one must not keep what it found: its files and its record
|
||||
// would bring back what the admin just removed. Keeping a result takes the read side, so
|
||||
// a purge waits for the results being kept and the next ones are refused.
|
||||
purgeMu sync.RWMutex
|
||||
purges uint64
|
||||
tidied sync.Map // the folders whose old temporary files were already removed
|
||||
}
|
||||
|
||||
// NewService builds a Service that keeps its files under dir.
|
||||
func NewService(dir string, store Store, src Source) *Service {
|
||||
return &Service{dir: dir, store: store, src: src, embedded: taglib.ReadImage, now: time.Now,
|
||||
slots: make(chan struct{}, MaxParallelBuilds), artistSl: make(chan struct{}, MaxParallelBuilds)}
|
||||
slots: make(chan struct{}, MaxParallelBuilds), artistSl: make(chan struct{}, MaxParallelBuilds),
|
||||
buildTimeout: buildTimeout}
|
||||
}
|
||||
|
||||
// SetArtistSource tells where pictures of artists come from. Without one, they are
|
||||
|
|
@ -255,7 +264,7 @@ func (s *Service) open(kind string, id int64, size Size, rec Record) (*Cover, er
|
|||
// build looks for a picture, stores it and records the outcome. If a file of the
|
||||
// album changed while it was working, what it found may be out of date: it starts again.
|
||||
func (s *Service) build(albumID int64) (Record, error) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), buildTimeout)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), s.buildTimeout)
|
||||
defer cancel()
|
||||
var rec Record
|
||||
for attempt := 0; attempt < 3; attempt++ {
|
||||
|
|
|
|||
|
|
@ -64,9 +64,11 @@ type Getter interface {
|
|||
|
||||
// Client talks to Wikidata and Commons. It is safe for concurrent use.
|
||||
type Client struct {
|
||||
http Getter
|
||||
wikidata string
|
||||
commons string
|
||||
http Getter
|
||||
downloader Getter // for the pictures themselves; the API client when nil
|
||||
wikidata string
|
||||
commons string
|
||||
allowAny bool // tests only: the test servers are not on a Wikimedia host
|
||||
}
|
||||
|
||||
// New builds a Client that sends its requests through g, to wikidata and commons
|
||||
|
|
@ -75,6 +77,14 @@ func New(g Getter, wikidata, commons string) *Client {
|
|||
return &Client{http: g, wikidata: wikidata, commons: commons}
|
||||
}
|
||||
|
||||
// WithDownloader makes the pictures themselves be downloaded through g instead of the
|
||||
// client of the API calls. They have their own limits (a picture is far bigger than an
|
||||
// answer) and their own breaker: a picture host that misbehaves must not stop the API.
|
||||
func (c *Client) WithDownloader(g Getter) *Client {
|
||||
c.downloader = g
|
||||
return c
|
||||
}
|
||||
|
||||
// File is a Commons picture and what has to be shown with it.
|
||||
type File struct {
|
||||
Name string // the file name, without the "File:"
|
||||
|
|
@ -228,10 +238,20 @@ func (c *Client) Info(ctx context.Context, name string) (File, error) {
|
|||
// Download reads a picture from a Wikimedia picture host. Any other address is refused
|
||||
// without a request.
|
||||
func (c *Client) Download(ctx context.Context, pictureURL string) ([]byte, error) {
|
||||
if !allowedPictureURL(pictureURL) {
|
||||
if !c.allowAny && !allowedPictureURL(pictureURL) {
|
||||
return nil, ErrBadRequest
|
||||
}
|
||||
return c.get(ctx, pictureURL)
|
||||
g := c.downloader
|
||||
if g == nil {
|
||||
g = c.http
|
||||
}
|
||||
body, err := c.getWith(ctx, g, pictureURL)
|
||||
// A redirect (never followed) or a picture over the limit is a fault of that picture,
|
||||
// not an outage: there is no usable picture to show.
|
||||
if errors.Is(err, outbound.ErrRedirect) || errors.Is(err, outbound.ErrResponseTooLarge) {
|
||||
return nil, fmt.Errorf("%w: the picture host answered in a way that cannot be used", ErrUnusable)
|
||||
}
|
||||
return body, err
|
||||
}
|
||||
|
||||
// Probe asks about one public item, to find out whether the service answers. It is a
|
||||
|
|
@ -290,8 +310,10 @@ var tagRe = regexp.MustCompile(`<[^>]*>`)
|
|||
// entities decoded, control characters dropped, spaces collapsed, at most maxAuthorLen.
|
||||
func plainText(s string) string {
|
||||
s = html.UnescapeString(tagRe.ReplaceAllString(s, ""))
|
||||
// Control characters, and the invisible ones that reorder or hide text (a right-to-left
|
||||
// override, a zero-width space): they could make an author read as somebody else.
|
||||
s = strings.Map(func(r rune) rune {
|
||||
if unicode.IsControl(r) && !unicode.IsSpace(r) {
|
||||
if (unicode.IsControl(r) && !unicode.IsSpace(r)) || unicode.Is(unicode.Cf, r) {
|
||||
return -1
|
||||
}
|
||||
return r
|
||||
|
|
@ -329,7 +351,11 @@ func allowedPictureURL(raw string) bool {
|
|||
|
||||
// get sends one request and returns the body of a successful answer.
|
||||
func (c *Client) get(ctx context.Context, rawURL string) ([]byte, error) {
|
||||
resp, err := c.http.Get(ctx, rawURL)
|
||||
return c.getWith(ctx, c.http, rawURL)
|
||||
}
|
||||
|
||||
func (c *Client) getWith(ctx context.Context, g Getter, rawURL string) ([]byte, error) {
|
||||
resp, err := g.Get(ctx, rawURL)
|
||||
if err != nil {
|
||||
if ctx.Err() != nil {
|
||||
return nil, ctx.Err()
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
package commons
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"net/http"
|
||||
|
|
@ -212,6 +213,15 @@ func TestTheLicenceAddressIsKeptOnlyOnCreativeCommons(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
func TestTheAuthorHasNoInvisibleOrReorderingCharacters(t *testing.T) {
|
||||
// A right-to-left override or a zero-width character can make a name read as another.
|
||||
body := strings.Replace(infoBody, `<a rel=\"nofollow\" class=\"external text\" href=\"https://www.flickr.com/photos/raph_ph/\">Raph_PH</a>`, `Ra\u202ephH\u200b_\u2066x`, 1)
|
||||
got, err := newServer(t, 200, body).client().Info(context.Background(), "Band.jpg")
|
||||
if err != nil || got.Author != "RaphH_x" {
|
||||
t.Errorf("author = %q err = %v", got.Author, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheAuthorIsPlainText(t *testing.T) {
|
||||
for artist, want := range map[string]string{
|
||||
`<a href=\"https://x\">Raph_PH</a>`: "Raph_PH",
|
||||
|
|
@ -284,6 +294,41 @@ func TestDownloadOnlyFromWikimediaPictureHosts(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
// What the picture host does wrong is a problem of that picture, not of the service: it must
|
||||
// not be taken for an outage, and it must not count against the breaker of the API client.
|
||||
func TestDownloadTreatsARedirectOrAHugePictureAsUnusable(t *testing.T) {
|
||||
redirect := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
http.Redirect(w, r, "https://evil.example/", http.StatusFound)
|
||||
}))
|
||||
defer redirect.Close()
|
||||
huge := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
_, _ = w.Write(bytes.Repeat([]byte{1}, 5000))
|
||||
}))
|
||||
defer huge.Close()
|
||||
for name, srv := range map[string]*httptest.Server{"redirect": redirect, "huge": huge} {
|
||||
dl := outbound.New(outbound.Options{UserAgent: "Sonda/test ( https://example.org )", Retries: outbound.NoRetries, MaxBody: 1000})
|
||||
c := New(nil, "", "").WithDownloader(dl)
|
||||
c.allowAny = true // the test servers are not on a Wikimedia host
|
||||
if _, err := c.Download(context.Background(), srv.URL+"/a.jpg"); !errors.Is(err, ErrUnusable) {
|
||||
t.Errorf("%s: err = %v, want ErrUnusable", name, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDownloadUsesItsOwnClient(t *testing.T) {
|
||||
api := newServer(t, 200, `{"claims":{}}`)
|
||||
pics := newServer(t, 200, "PICTURE")
|
||||
c := api.client().WithDownloader(outbound.New(outbound.Options{UserAgent: "Sonda/test ( https://example.org )", Retries: outbound.NoRetries}))
|
||||
c.allowAny = true
|
||||
got, err := c.Download(context.Background(), pics.URL+"/a.jpg")
|
||||
if err != nil || string(got) != "PICTURE" {
|
||||
t.Fatalf("got %q err %v", got, err)
|
||||
}
|
||||
if len(api.requests()) != 0 || len(pics.requests()) != 1 {
|
||||
t.Errorf("api saw %v, pictures saw %v: the download must go through its own client", api.requests(), pics.requests())
|
||||
}
|
||||
}
|
||||
|
||||
func TestErrorsDoNotCopyTheProvidersText(t *testing.T) {
|
||||
secret := "SECRET-FROM-THE-PROVIDER"
|
||||
_, err := newServer(t, 200, `{"error":{"code":"x","info":"`+secret+`"}}`).client().FileName(context.Background(), "Q1")
|
||||
|
|
|
|||
Loading…
Reference in a new issue