From 9e85d4d1ea9ce526948ff7014333261a3793275c Mon Sep 17 00:00:00 2001 From: tgiachi Date: Wed, 11 Nov 2026 22:12:49 +0100 Subject: [PATCH] fix(images): outages are 502, purges are not undone, bad pictures are not outages - A timeout inside the provider (the outbound client's own deadline, which wraps context.DeadlineExceeded) was mistaken for the caller leaving and came out as a 500, so an older stored picture was not served. The decision now uses the caller's own context, and a build that runs out of its two minutes is an outage (502) too. - A purge of the Commons data could be undone by a read that was already running: the row was deleted, so the generation guard started from zero, and the files were written after the files were removed. Pictures are now kept only under a purge counter; BeginArtistPurge is called before the purge and a read that started earlier starts again, or is refused if the provider is off by then. - A redirect or an oversized picture from the picture host is a fault of that picture (no picture), not an outage, and the pictures are downloaded through a client of their own so that one bad file cannot open the breaker of the API calls. - The author drops the invisible characters that reorder or hide text. Artist pictures are decoded only up to 4 million pixels. - Tests: an outage that ends in a deadline, a build that times out, a purge during a read, a redirect and a huge picture from the host, a separate download client. --- cmd/sondad/main.go | 12 ++- internal/api/openapi/openapi.yaml | 3 +- internal/images/artist.go | 86 ++++++++++++++++++---- internal/images/artist_test.go | 84 +++++++++++++++++++-- internal/images/commons_source.go | 22 +++--- internal/images/commons_source_test.go | 33 ++++++++- internal/images/process.go | 8 +- internal/images/service.go | 33 ++++++--- internal/providers/commons/commons.go | 40 ++++++++-- internal/providers/commons/commons_test.go | 45 +++++++++++ 10 files changed, 308 insertions(+), 58 deletions(-) diff --git a/cmd/sondad/main.go b/cmd/sondad/main.go index 9298c17..4cb8c06 100644 --- a/cmd/sondad/main.go +++ b/cmd/sondad/main.go @@ -59,6 +59,7 @@ const ( // commonsMaxBody leaves room for a 960 pixel copy of a picture, which is some hundred kilobytes. const ( commonsMinInterval = 300 * time.Millisecond + commonsAPIMaxBody = 1 << 20 commonsMaxBody = 8 << 20 ) @@ -159,11 +160,17 @@ func run() error { return wikipediaClient.Probe(ctx) }) // Pictures of artists from Wikimedia Commons, through Wikidata, under free licences only. + // The pictures are downloaded through a client of their own: they are far bigger than an + // answer, and a picture host that misbehaves must not open the breaker of the API. commonsClient := commons.New(outbound.New(outbound.Options{ + UserAgentFunc: providerSettings.UserAgentFor(providers.Commons, version), + MinInterval: commonsMinInterval, + MaxBody: commonsAPIMaxBody, + }), commons.WikidataURL, commons.CommonsURL).WithDownloader(outbound.New(outbound.Options{ UserAgentFunc: providerSettings.UserAgentFor(providers.Commons, version), MinInterval: commonsMinInterval, MaxBody: commonsMaxBody, - }), commons.WikidataURL, commons.CommonsURL) + })) providerSettings.SetChecker(providers.Commons, func(ctx context.Context, _ string) error { return commonsClient.Probe(ctx) }) @@ -177,6 +184,9 @@ func run() error { Artists: discoveryRepo, Settings: providerSettings, MusicBrainz: musicbrainzClient, Commons: commonsClient, }) providerSettings.SetPurger(func(ctx context.Context, name string) (providers.Purged, error) { + if name == providers.Commons { + pictures.BeginArtistPurge() // a picture being read right now must not be kept + } out, err := discoveryRepo.Purge(ctx, name) if err != nil || name != providers.Commons { return out, err diff --git a/internal/api/openapi/openapi.yaml b/internal/api/openapi/openapi.yaml index e08ca70..7347b32 100644 --- a/internal/api/openapi/openapi.yaml +++ b/internal/api/openapi/openapi.yaml @@ -1088,7 +1088,8 @@ paths: Whose the picture is and under which licence, to be shown next to it (the licences ask for it). The author is plain text and may be empty; `license_url` may be empty when the licence has no address; `page_url` is the page of the file on Commons, where everything is. - It answers like `GET /artists/{id}/image` (same errors), because the credit exists only with a picture. + It answers like `GET /artists/{id}/image` (same errors), because the credit exists only with a + picture: the first request for an artist can take as long as the picture does (several seconds). security: [{bearerAuth: []}] responses: '200': diff --git a/internal/images/artist.go b/internal/images/artist.go index c650dd8..15468f0 100644 --- a/internal/images/artist.go +++ b/internal/images/artist.go @@ -70,13 +70,33 @@ func (s *Service) ArtistImage(ctx context.Context, artistID int64, size Size) (* return s.open(kindArtist, artistID, size, built) } +// MaxArtistPixels is the largest copy of an artist's picture that is decoded. The copy +// asked from Commons is 960 pixels wide, so this leaves room for a very tall one. +const MaxArtistPixels = 4_000_000 + +// BeginArtistPurge must be called before the pictures of artists are taken back (their +// records forgotten and their files removed). A read that started before it is not kept: +// it starts again, and is refused if the provider is by then off. +func (s *Service) BeginArtistPurge() { + s.purgeMu.Lock() + s.purges++ + s.purgeMu.Unlock() +} + +func (s *Service) purgeCount() uint64 { + s.purgeMu.RLock() + defer s.purgeMu.RUnlock() + return s.purges +} + // buildArtist asks the source, stores the picture and records the outcome. If the -// record was cleared while it worked, what it found may be out of date: it starts again. +// record was cleared or the pictures were taken back while it worked, what it found may +// be out of date: it starts again. func (s *Service) buildArtist(artistID int64) (Record, error) { if s.artists == nil { return Record{}, ErrProviderDisabled } - ctx, cancel := context.WithTimeout(context.Background(), buildTimeout) + ctx, cancel := context.WithTimeout(context.Background(), s.buildTimeout) defer cancel() var rec Record for attempt := 0; attempt < 3; attempt++ { @@ -84,14 +104,27 @@ func (s *Service) buildArtist(artistID int64) (Record, error) { if err != nil { return Record{}, fmt.Errorf("read picture record: %w", err) } - rec, err = s.searchArtist(ctx, artistID) + epoch := s.purgeCount() + var pic *Processed + rec, pic, err = s.searchArtist(ctx, artistID) + if errors.Is(err, errPurged) { + continue + } if err != nil { + // Nobody calls this context: when it ends it is the build that ran out of time, + // which is an outage of the provider and not something to hide as an internal error. + if ctx.Err() != nil && (errors.Is(err, context.DeadlineExceeded) || errors.Is(err, context.Canceled)) { + err = fmt.Errorf("%w: %w", ErrSourceUnavailable, err) + } return Record{}, err } rec.Gen = cur.Gen - saved, err := s.store.Save(ctx, kindArtist, artistID, rec) + saved, err := s.saveArtist(ctx, artistID, rec, pic, epoch) + if errors.Is(err, errPurged) { + continue + } if err != nil { - return Record{}, fmt.Errorf("save picture record: %w", err) + return Record{}, err } if saved { return rec, nil @@ -100,27 +133,30 @@ func (s *Service) buildArtist(artistID int64) (Record, error) { return rec, nil } -func (s *Service) searchArtist(ctx context.Context, artistID int64) (Record, error) { +// errPurged says the pictures were taken back since the build began. +var errPurged = errors.New("the pictures were taken back while this was being built") + +// searchArtist asks the source and, for a picture, prepares it (nil when there is none). +// The files are written by saveArtist, together with the record, so that a purge cannot +// fall between them. +func (s *Service) searchArtist(ctx context.Context, artistID int64) (Record, *Processed, error) { select { case s.artistSl <- struct{}{}: defer func() { <-s.artistSl }() case <-ctx.Done(): - return Record{}, ctx.Err() + return Record{}, nil, ctx.Err() } none := Record{Status: StatusNone, CheckedAt: s.now()} found, err := s.artists.ArtistImage(ctx, artistID) switch { case errors.Is(err, ErrNoImage): - return none, nil + return none, nil, nil case err != nil: - return Record{}, err + return Record{}, nil, err } - p, err := Process(found.Data) + p, err := ProcessLimited(found.Data, MaxArtistPixels) if err != nil { - return none, nil // what the provider sent is not a picture: there is none to show - } - if err := s.write(kindArtist, artistID, p); err != nil { - return Record{}, err + return none, nil, nil // what the provider sent is not a usable picture: there is none to show } // The credit is part of the identity: the same picture under another author or licence // must not be taken for the same one by a client that kept the old credit. @@ -130,7 +166,27 @@ func (s *Service) searchArtist(ctx context.Context, artistID int64) (Record, err return Record{ Status: StatusOK, Source: SourceCommons, Credit: found.Credit, ETag: fmt.Sprintf("%x", h.Sum(nil)[:8]), Width: p.Width, Height: p.Height, CheckedAt: none.CheckedAt, - }, nil + }, &p, nil +} + +// saveArtist keeps what was found: the files and the record, unless the pictures were +// taken back since the build began (errPurged) or a change moved the record's generation. +func (s *Service) saveArtist(ctx context.Context, artistID int64, rec Record, pic *Processed, epoch uint64) (bool, error) { + s.purgeMu.RLock() + defer s.purgeMu.RUnlock() + if s.purges != epoch { + return false, errPurged + } + if pic != nil { + if err := s.write(kindArtist, artistID, *pic); err != nil { + return false, err + } + } + saved, err := s.store.Save(ctx, kindArtist, artistID, rec) + if err != nil { + return false, fmt.Errorf("save picture record: %w", err) + } + return saved, nil } // RemoveArtistFiles deletes the stored pictures of artists, and only those, and says how diff --git a/internal/images/artist_test.go b/internal/images/artist_test.go index 98b012b..ef90483 100644 --- a/internal/images/artist_test.go +++ b/internal/images/artist_test.go @@ -21,15 +21,24 @@ type fakeArtists struct { err error calls int32 gate chan struct{} + gate2 bool // the gate is never opened for this call: it waits for the context instead } -func (f *fakeArtists) ArtistImage(_ context.Context, id int64) (Found, error) { +func (f *fakeArtists) ArtistImage(ctx context.Context, id int64) (Found, error) { atomic.AddInt32(&f.calls, 1) f.mu.Lock() gate, found, err := f.gate, f.found, f.err f.mu.Unlock() if gate != nil { - <-gate + if f.gate2 { + select { + case <-gate: + case <-ctx.Done(): + return Found{}, ctx.Err() + } + } else { + <-gate + } } return found, err } @@ -205,18 +214,77 @@ func TestManyAsksForAnArtistPictureAskTheProviderOnce(t *testing.T) { } } -func TestAChangeDuringTheReadOfAnArtistPictureIsNotUndoneByIt(t *testing.T) { +func TestAPurgeDuringTheReadIsNotUndoneByIt(t *testing.T) { + r := newArtistRig(t) + r.src.gate = make(chan struct{}) + done := make(chan error, 1) + go func() { _, err := r.svc.ArtistImage(context.Background(), 7, Large); done <- err }() + time.Sleep(50 * time.Millisecond) // the read has the picture and is about to keep it + r.svc.BeginArtistPurge() // the admin takes the provider's data back... + r.src.set(Found{}, ErrProviderDisabled) // ...and switches it off + close(r.src.gate) + if err := <-done; !errors.Is(err, ErrProviderDisabled) { + t.Fatalf("err = %v, want the read to be refused once the provider is off", err) + } + if len(r.store.recs) != 0 { + t.Errorf("records = %v: what the read found before the purge came back", r.store.recs) + } + if entries, _ := os.ReadDir(filepath.Join(r.data, "images", "artist")); len(entries) != 0 { + t.Errorf("%d files left by a read that started before the purge", len(entries)) + } +} + +func TestAReadThatStartedBeforeAPurgeStartsAgainIfTheProviderIsStillOn(t *testing.T) { r := newArtistRig(t) r.src.gate = make(chan struct{}) done := make(chan error, 1) go func() { _, err := r.svc.ArtistImage(context.Background(), 7, Large); done <- err }() time.Sleep(50 * time.Millisecond) - r.store.clear("artist", 7) // for example the purge of the provider + r.svc.BeginArtistPurge() close(r.src.gate) - <-done - // What the read that started before the change found was not kept as the answer. - if rec := r.store.recs["artist/7"]; rec.Status == StatusOK && rec.Gen == 0 { - t.Errorf("record = %+v: an out of date build was saved over the change", rec) + if err := <-done; err != nil { + t.Fatalf("err = %v", err) + } + if n := atomic.LoadInt32(&r.src.calls); n != 2 { + t.Errorf("the provider was asked %d times, want 2: the first answer was from before the purge", n) + } + if rec := r.store.recs["artist/7"]; rec.Status != StatusOK { + t.Errorf("record = %+v", rec) + } +} + +func TestABuildThatTakesTooLongIsAnOutageNotAnInternalError(t *testing.T) { + r := newArtistRig(t) + _, old := r.get(t, 7, Large) // a picture is stored + r.now = r.now.Add(ArtistTTL + time.Hour) + r.svc.buildTimeout = 40 * time.Millisecond + r.src.gate = make(chan struct{}) // never opened during the build + r.src.gate2 = true + c, again := r.get(t, 7, Large) + if !bytes.Equal(old, again) || c.Credit != credit { + t.Error("the old picture was not served when the build timed out") + } + // With nothing stored the caller is told it is an outage (502), not a failure of ours (500). + r2 := newArtistRig(t) + r2.svc.buildTimeout = 40 * time.Millisecond + r2.src.gate = make(chan struct{}) + r2.src.gate2 = true + if _, err := r2.svc.ArtistImage(context.Background(), 8, Large); !errors.Is(err, ErrSourceUnavailable) { + t.Errorf("err = %v, want ErrSourceUnavailable", err) + } + close(r.src.gate) + close(r2.src.gate) +} + +func TestAHugePictureOfAnArtistIsRefused(t *testing.T) { + r := newArtistRig(t) + r.src.set(Found{Data: jpegBytes(t, 2500, 2500, color.White), Credit: credit}, nil) // 6 million pixels + if _, err := r.svc.ArtistImage(context.Background(), 7, Large); !errors.Is(err, ErrNoImage) { + t.Errorf("err = %v, want ErrNoImage: a copy of 960 pixels is never that big", err) + } + // The same picture is fine as the cover of an album (the limit there is larger). + if _, err := Process(r.src.found.Data); err != nil { + t.Errorf("Process: %v", err) } } diff --git a/internal/images/commons_source.go b/internal/images/commons_source.go index bfbcac3..d75c416 100644 --- a/internal/images/commons_source.go +++ b/internal/images/commons_source.go @@ -58,11 +58,13 @@ func (c *CommonsSource) active(ctx context.Context) (bool, error) { return true, nil } -// failed turns what went wrong into ErrSourceUnavailable, keeping the cause. A request -// that was cancelled is not a failure of the provider and is returned as it is. -func failed(err error) error { - if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { - return err +// failed turns what went wrong into ErrSourceUnavailable, keeping the cause. When the +// caller's own context has ended (it left, or ran out of time) that is returned instead: +// it is not the provider's fault. A deadline buried in the error is not a sign of that, since +// the outbound client's own timeout produces one when the provider is merely slow. +func failed(ctx context.Context, err error) error { + if ctxErr := ctx.Err(); ctxErr != nil { + return ctxErr } return fmt.Errorf("%w: %w", ErrSourceUnavailable, err) } @@ -90,7 +92,7 @@ func (c *CommonsSource) ArtistImage(ctx context.Context, artistID int64) (Found, case errors.Is(err, musicbrainz.ErrNotFound), errors.Is(err, musicbrainz.ErrBadRequest): return Found{}, ErrNoImage case err != nil: - return Found{}, failed(err) + return Found{}, failed(ctx, err) case item == "": return Found{}, ErrNoImage } @@ -99,7 +101,7 @@ func (c *CommonsSource) ArtistImage(ctx context.Context, artistID int64) (Found, case errors.Is(err, commons.ErrNotFound), errors.Is(err, commons.ErrBadRequest): return Found{}, ErrNoImage case err != nil: - return Found{}, failed(err) + return Found{}, failed(ctx, err) case name == "": return Found{}, ErrNoImage } @@ -109,7 +111,7 @@ func (c *CommonsSource) ArtistImage(ctx context.Context, artistID int64) (Found, errors.Is(err, commons.ErrUnusable), errors.Is(err, commons.ErrBadRequest): return Found{}, ErrNoImage case err != nil: - return Found{}, failed(err) + return Found{}, failed(ctx, err) } // The admin may have switched the provider off while the chain was being followed: // do not download, and do not hand over, what they no longer allow. @@ -120,10 +122,10 @@ func (c *CommonsSource) ArtistImage(ctx context.Context, artistID int64) (Found, } data, err := c.Commons.Download(ctx, file.ThumbURL) switch { - case errors.Is(err, commons.ErrBadRequest), errors.Is(err, commons.ErrNotFound): + case errors.Is(err, commons.ErrBadRequest), errors.Is(err, commons.ErrNotFound), errors.Is(err, commons.ErrUnusable): return Found{}, ErrNoImage case err != nil: - return Found{}, failed(err) + return Found{}, failed(ctx, err) } if on, err := c.active(ctx); err != nil { return Found{}, err diff --git a/internal/images/commons_source_test.go b/internal/images/commons_source_test.go index ddf3894..529faa1 100644 --- a/internal/images/commons_source_test.go +++ b/internal/images/commons_source_test.go @@ -3,6 +3,7 @@ package images import ( "context" "errors" + "fmt" "sync" "testing" @@ -194,10 +195,38 @@ func TestSwitchedOffWhileItWasWorkingNothingIsReturned(t *testing.T) { } } -func TestACancelledRequestIsNotAFailureOfTheProvider(t *testing.T) { +func TestAnOutageThatEndsInADeadlineIsStillAnOutage(t *testing.T) { + // The outbound client's own timeout wraps context.DeadlineExceeded inside ErrUnavailable. It + // is a failure of the provider (so an older picture can be served), not a cancellation. + slow := fmt.Errorf("%w: %w", commons.ErrUnavailable, context.DeadlineExceeded) + for name, tt := range map[string]func(*chain){ + "musicbrainz": func(c *chain) { c.mb.err = fmt.Errorf("%w: %w", musicbrainz.ErrUnavailable, context.DeadlineExceeded) }, + "wikidata": func(c *chain) { c.cm.nameErr = slow }, + "commons": func(c *chain) { c.cm.infoErr = slow }, + "the download": func(c *chain) { c.cm.loadErr = slow }, + } { + c := newChain() + tt(c) + if _, err := c.src.ArtistImage(context.Background(), 1); !errors.Is(err, ErrSourceUnavailable) { + t.Errorf("%s: err = %v, want ErrSourceUnavailable", name, err) + } + } +} + +func TestARequestThatTheCallerCancelledIsReturnedAsCancelled(t *testing.T) { c := newChain() + ctx, cancel := context.WithCancel(context.Background()) + c.mb.hook = cancel // the caller leaves while MusicBrainz is being asked c.mb.err = context.Canceled - if _, err := c.src.ArtistImage(context.Background(), 1); !errors.Is(err, context.Canceled) { + if _, err := c.src.ArtistImage(ctx, 1); !errors.Is(err, context.Canceled) || errors.Is(err, ErrSourceUnavailable) { t.Errorf("err = %v, want the cancellation itself", err) } } + +func TestAPictureThatCannotBeUsedFromTheHostIsNoPicture(t *testing.T) { + c := newChain() + c.cm.loadErr = commons.ErrUnusable // a redirect or a picture over the limit + if _, err := c.src.ArtistImage(context.Background(), 1); !errors.Is(err, ErrNoImage) { + t.Errorf("err = %v, want ErrNoImage", err) + } +} diff --git a/internal/images/process.go b/internal/images/process.go index ceba78c..1b3716a 100644 --- a/internal/images/process.go +++ b/internal/images/process.go @@ -45,7 +45,11 @@ type Processed struct { // Process decodes a GIF, JPEG or PNG and returns it scaled down to fit LargeEdge and // SmallEdge (never enlarged), on a white background where it was transparent. The // same input always gives the same bytes. -func Process(data []byte) (Processed, error) { +func Process(data []byte) (Processed, error) { return ProcessLimited(data, MaxPixels) } + +// ProcessLimited is Process with a smaller limit on the number of pixels, for pictures that +// are known to be small (a 960 pixel copy is never millions of pixels high). +func ProcessLimited(data []byte, maxPixels int64) (Processed, error) { if len(data) == 0 || len(data) > MaxSourceBytes { return Processed{}, fmt.Errorf("%w: %d bytes", ErrUnusable, len(data)) } @@ -53,7 +57,7 @@ func Process(data []byte) (Processed, error) { if err != nil { return Processed{}, fmt.Errorf("%w: %v", ErrUnusable, err) } - if cfg.Width <= 0 || cfg.Height <= 0 || int64(cfg.Width)*int64(cfg.Height) > MaxPixels { // int64: a 32-bit int would overflow + if cfg.Width <= 0 || cfg.Height <= 0 || int64(cfg.Width)*int64(cfg.Height) > maxPixels { // int64: a 32-bit int would overflow return Processed{}, fmt.Errorf("%w: %dx%d pixels", ErrUnusable, cfg.Width, cfg.Height) } src, _, err := image.Decode(bytes.NewReader(data)) diff --git a/internal/images/service.go b/internal/images/service.go index 007f4cd..8abf91c 100644 --- a/internal/images/service.go +++ b/internal/images/service.go @@ -157,22 +157,31 @@ type Cover struct { // Service builds covers from the user's own files, on the first request, and keeps // them in the data folder. type Service struct { - dir string - store Store - src Source - embedded func(path string) ([]byte, error) - now func() time.Time - flight singleflight.Group - slots chan struct{} // one per cover build allowed at the same time - artistSl chan struct{} // the same for artist pictures, which wait for the network - artists ArtistSource - tidied sync.Map // the folders whose old temporary files were already removed + dir string + store Store + src Source + embedded func(path string) ([]byte, error) + now func() time.Time + flight singleflight.Group + slots chan struct{} // one per cover build allowed at the same time + artistSl chan struct{} // the same for artist pictures, which wait for the network + artists ArtistSource + buildTimeout time.Duration // how long one build may take + + // purges counts the times the provider's pictures were taken back (BeginArtistPurge). A + // build that started before one must not keep what it found: its files and its record + // would bring back what the admin just removed. Keeping a result takes the read side, so + // a purge waits for the results being kept and the next ones are refused. + purgeMu sync.RWMutex + purges uint64 + tidied sync.Map // the folders whose old temporary files were already removed } // NewService builds a Service that keeps its files under dir. func NewService(dir string, store Store, src Source) *Service { return &Service{dir: dir, store: store, src: src, embedded: taglib.ReadImage, now: time.Now, - slots: make(chan struct{}, MaxParallelBuilds), artistSl: make(chan struct{}, MaxParallelBuilds)} + slots: make(chan struct{}, MaxParallelBuilds), artistSl: make(chan struct{}, MaxParallelBuilds), + buildTimeout: buildTimeout} } // SetArtistSource tells where pictures of artists come from. Without one, they are @@ -255,7 +264,7 @@ func (s *Service) open(kind string, id int64, size Size, rec Record) (*Cover, er // build looks for a picture, stores it and records the outcome. If a file of the // album changed while it was working, what it found may be out of date: it starts again. func (s *Service) build(albumID int64) (Record, error) { - ctx, cancel := context.WithTimeout(context.Background(), buildTimeout) + ctx, cancel := context.WithTimeout(context.Background(), s.buildTimeout) defer cancel() var rec Record for attempt := 0; attempt < 3; attempt++ { diff --git a/internal/providers/commons/commons.go b/internal/providers/commons/commons.go index d769580..71255ef 100644 --- a/internal/providers/commons/commons.go +++ b/internal/providers/commons/commons.go @@ -64,9 +64,11 @@ type Getter interface { // Client talks to Wikidata and Commons. It is safe for concurrent use. type Client struct { - http Getter - wikidata string - commons string + http Getter + downloader Getter // for the pictures themselves; the API client when nil + wikidata string + commons string + allowAny bool // tests only: the test servers are not on a Wikimedia host } // New builds a Client that sends its requests through g, to wikidata and commons @@ -75,6 +77,14 @@ func New(g Getter, wikidata, commons string) *Client { return &Client{http: g, wikidata: wikidata, commons: commons} } +// WithDownloader makes the pictures themselves be downloaded through g instead of the +// client of the API calls. They have their own limits (a picture is far bigger than an +// answer) and their own breaker: a picture host that misbehaves must not stop the API. +func (c *Client) WithDownloader(g Getter) *Client { + c.downloader = g + return c +} + // File is a Commons picture and what has to be shown with it. type File struct { Name string // the file name, without the "File:" @@ -228,10 +238,20 @@ func (c *Client) Info(ctx context.Context, name string) (File, error) { // Download reads a picture from a Wikimedia picture host. Any other address is refused // without a request. func (c *Client) Download(ctx context.Context, pictureURL string) ([]byte, error) { - if !allowedPictureURL(pictureURL) { + if !c.allowAny && !allowedPictureURL(pictureURL) { return nil, ErrBadRequest } - return c.get(ctx, pictureURL) + g := c.downloader + if g == nil { + g = c.http + } + body, err := c.getWith(ctx, g, pictureURL) + // A redirect (never followed) or a picture over the limit is a fault of that picture, + // not an outage: there is no usable picture to show. + if errors.Is(err, outbound.ErrRedirect) || errors.Is(err, outbound.ErrResponseTooLarge) { + return nil, fmt.Errorf("%w: the picture host answered in a way that cannot be used", ErrUnusable) + } + return body, err } // Probe asks about one public item, to find out whether the service answers. It is a @@ -290,8 +310,10 @@ var tagRe = regexp.MustCompile(`<[^>]*>`) // entities decoded, control characters dropped, spaces collapsed, at most maxAuthorLen. func plainText(s string) string { s = html.UnescapeString(tagRe.ReplaceAllString(s, "")) + // Control characters, and the invisible ones that reorder or hide text (a right-to-left + // override, a zero-width space): they could make an author read as somebody else. s = strings.Map(func(r rune) rune { - if unicode.IsControl(r) && !unicode.IsSpace(r) { + if (unicode.IsControl(r) && !unicode.IsSpace(r)) || unicode.Is(unicode.Cf, r) { return -1 } return r @@ -329,7 +351,11 @@ func allowedPictureURL(raw string) bool { // get sends one request and returns the body of a successful answer. func (c *Client) get(ctx context.Context, rawURL string) ([]byte, error) { - resp, err := c.http.Get(ctx, rawURL) + return c.getWith(ctx, c.http, rawURL) +} + +func (c *Client) getWith(ctx context.Context, g Getter, rawURL string) ([]byte, error) { + resp, err := g.Get(ctx, rawURL) if err != nil { if ctx.Err() != nil { return nil, ctx.Err() diff --git a/internal/providers/commons/commons_test.go b/internal/providers/commons/commons_test.go index 578b49f..6a0edee 100644 --- a/internal/providers/commons/commons_test.go +++ b/internal/providers/commons/commons_test.go @@ -1,6 +1,7 @@ package commons import ( + "bytes" "context" "errors" "net/http" @@ -212,6 +213,15 @@ func TestTheLicenceAddressIsKeptOnlyOnCreativeCommons(t *testing.T) { } } +func TestTheAuthorHasNoInvisibleOrReorderingCharacters(t *testing.T) { + // A right-to-left override or a zero-width character can make a name read as another. + body := strings.Replace(infoBody, `Raph_PH`, `Ra\u202ephH\u200b_\u2066x`, 1) + got, err := newServer(t, 200, body).client().Info(context.Background(), "Band.jpg") + if err != nil || got.Author != "RaphH_x" { + t.Errorf("author = %q err = %v", got.Author, err) + } +} + func TestTheAuthorIsPlainText(t *testing.T) { for artist, want := range map[string]string{ `Raph_PH`: "Raph_PH", @@ -284,6 +294,41 @@ func TestDownloadOnlyFromWikimediaPictureHosts(t *testing.T) { } } +// What the picture host does wrong is a problem of that picture, not of the service: it must +// not be taken for an outage, and it must not count against the breaker of the API client. +func TestDownloadTreatsARedirectOrAHugePictureAsUnusable(t *testing.T) { + redirect := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Redirect(w, r, "https://evil.example/", http.StatusFound) + })) + defer redirect.Close() + huge := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write(bytes.Repeat([]byte{1}, 5000)) + })) + defer huge.Close() + for name, srv := range map[string]*httptest.Server{"redirect": redirect, "huge": huge} { + dl := outbound.New(outbound.Options{UserAgent: "Sonda/test ( https://example.org )", Retries: outbound.NoRetries, MaxBody: 1000}) + c := New(nil, "", "").WithDownloader(dl) + c.allowAny = true // the test servers are not on a Wikimedia host + if _, err := c.Download(context.Background(), srv.URL+"/a.jpg"); !errors.Is(err, ErrUnusable) { + t.Errorf("%s: err = %v, want ErrUnusable", name, err) + } + } +} + +func TestDownloadUsesItsOwnClient(t *testing.T) { + api := newServer(t, 200, `{"claims":{}}`) + pics := newServer(t, 200, "PICTURE") + c := api.client().WithDownloader(outbound.New(outbound.Options{UserAgent: "Sonda/test ( https://example.org )", Retries: outbound.NoRetries})) + c.allowAny = true + got, err := c.Download(context.Background(), pics.URL+"/a.jpg") + if err != nil || string(got) != "PICTURE" { + t.Fatalf("got %q err %v", got, err) + } + if len(api.requests()) != 0 || len(pics.requests()) != 1 { + t.Errorf("api saw %v, pictures saw %v: the download must go through its own client", api.requests(), pics.requests()) + } +} + func TestErrorsDoNotCopyTheProvidersText(t *testing.T) { secret := "SECRET-FROM-THE-PROVIDER" _, err := newServer(t, 200, `{"error":{"code":"x","info":"`+secret+`"}}`).client().FileName(context.Background(), "Q1")