fix(images): outages are 502, purges are not undone, bad pictures are not outages
- A timeout inside the provider (the outbound client's own deadline, which wraps context.DeadlineExceeded) was mistaken for the caller leaving and came out as a 500, so an older stored picture was not served. The decision now uses the caller's own context, and a build that runs out of its two minutes is an outage (502) too. - A purge of the Commons data could be undone by a read that was already running: the row was deleted, so the generation guard started from zero, and the files were written after the files were removed. Pictures are now kept only under a purge counter; BeginArtistPurge is called before the purge and a read that started earlier starts again, or is refused if the provider is off by then. - A redirect or an oversized picture from the picture host is a fault of that picture (no picture), not an outage, and the pictures are downloaded through a client of their own so that one bad file cannot open the breaker of the API calls. - The author drops the invisible characters that reorder or hide text. Artist pictures are decoded only up to 4 million pixels. - Tests: an outage that ends in a deadline, a build that times out, a purge during a read, a redirect and a huge picture from the host, a separate download client.
This commit is contained in:
parent
e752d15536
commit
9e85d4d1ea
1 changed files with 308 additions and 58 deletions
|
|
@ -59,6 +59,7 @@ const (
|
|||
// commonsMaxBody leaves room for a 960 pixel copy of a picture, which is some hundred kilobytes.
|
||||
const (
|
||||
commonsMinInterval = 300 * time.Millisecond
|
||||
commonsAPIMaxBody = 1 << 20
|
||||
commonsMaxBody = 8 << 20
|
||||
)
|
||||
|
||||
|
|
@ -159,11 +160,17 @@ func run() error {
|
|||
return wikipediaClient.Probe(ctx)
|
||||
})
|
||||
// Pictures of artists from Wikimedia Commons, through Wikidata, under free licences only.
|
||||
// The pictures are downloaded through a client of their own: they are far bigger than an
|
||||
// answer, and a picture host that misbehaves must not open the breaker of the API.
|
||||
commonsClient := commons.New(outbound.New(outbound.Options{
|
||||
UserAgentFunc: providerSettings.UserAgentFor(providers.Commons, version),
|
||||
MinInterval: commonsMinInterval,
|
||||
MaxBody: commonsAPIMaxBody,
|
||||
}), commons.WikidataURL, commons.CommonsURL).WithDownloader(outbound.New(outbound.Options{
|
||||
UserAgentFunc: providerSettings.UserAgentFor(providers.Commons, version),
|
||||
MinInterval: commonsMinInterval,
|
||||
MaxBody: commonsMaxBody,
|
||||
}), commons.WikidataURL, commons.CommonsURL)
|
||||
}))
|
||||
providerSettings.SetChecker(providers.Commons, func(ctx context.Context, _ string) error {
|
||||
return commonsClient.Probe(ctx)
|
||||
})
|
||||
|
|
@ -177,6 +184,9 @@ func run() error {
|
|||
Artists: discoveryRepo, Settings: providerSettings, MusicBrainz: musicbrainzClient, Commons: commonsClient,
|
||||
})
|
||||
providerSettings.SetPurger(func(ctx context.Context, name string) (providers.Purged, error) {
|
||||
if name == providers.Commons {
|
||||
pictures.BeginArtistPurge() // a picture being read right now must not be kept
|
||||
}
|
||||
out, err := discoveryRepo.Purge(ctx, name)
|
||||
if err != nil || name != providers.Commons {
|
||||
return out, err
|
||||
|
|
|
|||
Loading…
Reference in a new issue