fix(providers): keep the key out of errors and make the outbound client robust
- fix: errors from the outbound client no longer carry the query string. Go puts the whole URL into a transport error, and a Last.fm key travels in the query, so any log of such an error would have written the key. The host and path stay for debugging and the cause can still be matched with errors.Is - fix: a request that gives up while waiting for its turn gives the turn back, and a turn is not taken at all when the wait would pass the caller's deadline. Before, a burst of impatient callers pushed the queue further away until no request ever reached the provider - fix: failures of requests that were already in flight when the provider died no longer double the breaker window each; one outage opens it once - fix: redirects and oversized answers count as failures, so a provider that keeps answering that way stops being asked - fix: sending an empty api_key to a provider that needs none (MusicBrainz) no longer turns it off - test: the query string in transport errors, refused connections and timeouts; cancelling during the wait; the deadline shorter than the wait; twelve concurrent failures; redirects and big answers; the empty key. Checked by removing each fix on purpose
This commit is contained in:
parent
9f4da2e77e
commit
e231860119
5 changed files with 268 additions and 8 deletions
|
|
@ -793,8 +793,8 @@ paths:
|
|||
Admin only. Send any of `enabled`, `api_key` and `contact`; fields you leave out are kept.
|
||||
|
||||
The key is write-only: it is never returned and never logged. Sending an empty `api_key`
|
||||
clears it and turns the provider off. A provider that needs a key (Last.fm) cannot be
|
||||
enabled without one. `contact` goes into the User-Agent sent to the provider (MusicBrainz
|
||||
clears it, and turns a provider that needs a key (Last.fm) off; a provider that needs none
|
||||
(MusicBrainz) is not affected. A provider that needs a key cannot be enabled without one. `contact` goes into the User-Agent sent to the provider (MusicBrainz
|
||||
asks for a URL or an e-mail address); empty means the project address.
|
||||
security: [{bearerAuth: []}]
|
||||
requestBody:
|
||||
|
|
|
|||
|
|
@ -204,6 +204,9 @@ func (c *Client) Get(ctx context.Context, rawURL string) (*Response, error) {
|
|||
resp, err := c.once(ctx, rawURL)
|
||||
switch {
|
||||
case errors.Is(err, ErrRedirect), errors.Is(err, ErrResponseTooLarge):
|
||||
// Not worth retrying, but a provider that keeps answering like this
|
||||
// must stop being asked.
|
||||
c.failed()
|
||||
return nil, err
|
||||
case err != nil:
|
||||
if ctx.Err() != nil {
|
||||
|
|
@ -270,7 +273,7 @@ func (c *Client) once(ctx context.Context, rawURL string) (*Response, error) {
|
|||
|
||||
resp, err := c.hc.Do(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, redactQuery(err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
|
|
@ -287,6 +290,23 @@ func (c *Client) once(ctx context.Context, rawURL string) (*Response, error) {
|
|||
return &Response{Status: resp.StatusCode, Header: resp.Header.Clone(), Body: body}, nil
|
||||
}
|
||||
|
||||
// redactQuery removes the query string, fragment and credentials from the URL
|
||||
// that net/http puts into its errors. A Last.fm key travels in the query, and
|
||||
// an error that reaches a log must not carry it. The host and path stay, for
|
||||
// debugging, and the error can still be matched with errors.Is.
|
||||
func redactQuery(err error) error {
|
||||
var ue *url.Error
|
||||
if !errors.As(err, &ue) {
|
||||
return err
|
||||
}
|
||||
if u, perr := url.Parse(ue.URL); perr == nil {
|
||||
ue.URL = u.Scheme + "://" + u.Host + u.Path
|
||||
} else {
|
||||
ue.URL = "(unreadable URL)"
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func retryable(status int) bool {
|
||||
switch status {
|
||||
case http.StatusTooManyRequests, http.StatusBadGateway, http.StatusServiceUnavailable, http.StatusGatewayTimeout:
|
||||
|
|
@ -322,7 +342,10 @@ func retryAfter(h http.Header, now time.Time) (time.Duration, bool) {
|
|||
return 0, false
|
||||
}
|
||||
|
||||
// throttle reserves the next free slot and waits for it.
|
||||
// throttle reserves the next free slot and waits for it. A slot is only taken
|
||||
// when the caller can use it before its deadline, and it is given back when the
|
||||
// caller gives up while waiting. Otherwise a burst of impatient callers would
|
||||
// push the queue further and further away until nobody ever reached the provider.
|
||||
func (c *Client) throttle(ctx context.Context) error {
|
||||
if c.minInterval <= 0 {
|
||||
return nil
|
||||
|
|
@ -333,11 +356,24 @@ func (c *Client) throttle(ctx context.Context) error {
|
|||
if at.Before(now) {
|
||||
at = now
|
||||
}
|
||||
c.next = at.Add(c.minInterval)
|
||||
wait := at.Sub(now)
|
||||
if dl, ok := ctx.Deadline(); ok && wait > 0 && time.Until(dl) < wait {
|
||||
c.mu.Unlock()
|
||||
return fmt.Errorf("%w: waiting %v for our turn would pass the deadline", context.DeadlineExceeded, wait)
|
||||
}
|
||||
taken := at.Add(c.minInterval)
|
||||
c.next = taken
|
||||
c.mu.Unlock()
|
||||
|
||||
if wait := at.Sub(now); wait > 0 {
|
||||
return c.clock.Sleep(ctx, wait)
|
||||
if wait > 0 {
|
||||
if err := c.clock.Sleep(ctx, wait); err != nil {
|
||||
c.mu.Lock()
|
||||
if c.next.Equal(taken) { // nobody queued behind us: hand the turn back
|
||||
c.next = at
|
||||
}
|
||||
c.mu.Unlock()
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
|
@ -367,6 +403,12 @@ func (c *Client) failed() {
|
|||
if c.failures < c.breakerThreshold {
|
||||
return
|
||||
}
|
||||
// A failure that arrives while the breaker is already open belongs to the
|
||||
// requests that were in flight when the provider died: the same outage, not
|
||||
// a new reason to wait longer. Only a probe after the window may extend it.
|
||||
if c.clock.Now().Before(c.openUntil) {
|
||||
return
|
||||
}
|
||||
if c.openFor == 0 {
|
||||
c.openFor = c.breakerOpenFor
|
||||
} else if c.openFor *= 2; c.openFor > c.breakerMaxOpen {
|
||||
|
|
|
|||
|
|
@ -496,3 +496,205 @@ func TestDefaultsAreSane(t *testing.T) {
|
|||
t.Errorf("user agent = %q", c.userAgent)
|
||||
}
|
||||
}
|
||||
|
||||
// ---- review fixes ----
|
||||
|
||||
// failingTransport makes every request fail the way a dead network does.
|
||||
type failingTransport struct{}
|
||||
|
||||
func (failingTransport) RoundTrip(*http.Request) (*http.Response, error) {
|
||||
return nil, errors.New("boom: network is down")
|
||||
}
|
||||
|
||||
// A Last.fm key travels in the query string. Go puts the whole URL into a
|
||||
// transport error, so the client must cut the query out before it goes anywhere.
|
||||
func TestTransportErrorsNeverCarryTheQueryString(t *testing.T) {
|
||||
const secret = "SECRETKEY0123456789abcdef"
|
||||
url := "https://ws.example.org/2.0/?method=artist.getSimilar&artist=Creed&api_key=" + secret
|
||||
|
||||
check := func(name string, err error) {
|
||||
t.Helper()
|
||||
if err == nil {
|
||||
t.Fatalf("%s: expected an error", name)
|
||||
}
|
||||
msg := err.Error()
|
||||
if strings.Contains(msg, secret) || strings.Contains(msg, "artist=Creed") || strings.Contains(msg, "api_key") {
|
||||
t.Errorf("%s: the error repeats the query string: %q", name, msg)
|
||||
}
|
||||
if !strings.Contains(msg, "ws.example.org") {
|
||||
t.Errorf("%s: the host should stay, to debug: %q", name, msg)
|
||||
}
|
||||
}
|
||||
|
||||
c := newClient(newClock(), func(o *Options) { o.Transport = failingTransport{}; o.Retries = NoRetries })
|
||||
_, err := c.Get(context.Background(), url)
|
||||
check("failing transport", err)
|
||||
if !strings.Contains(err.Error(), "boom") {
|
||||
t.Errorf("the underlying cause was lost: %q", err.Error())
|
||||
}
|
||||
|
||||
// A refused connection on a real port.
|
||||
srv := httptest.NewServer(http.NotFoundHandler())
|
||||
dead := srv.URL
|
||||
srv.Close()
|
||||
c = newClient(newClock(), func(o *Options) { o.Retries = NoRetries })
|
||||
_, err = c.Get(context.Background(), dead+"/x?api_key="+secret)
|
||||
if err == nil || strings.Contains(err.Error(), secret) {
|
||||
t.Errorf("refused connection: %v", err)
|
||||
}
|
||||
|
||||
// A timeout.
|
||||
slow, _ := server(t, func(w http.ResponseWriter, r *http.Request) { <-r.Context().Done() })
|
||||
c = newClient(newClock(), func(o *Options) { o.Timeout = 30 * time.Millisecond; o.Retries = NoRetries })
|
||||
_, err = c.Get(context.Background(), slow.URL+"/x?api_key="+secret)
|
||||
if err == nil || strings.Contains(err.Error(), secret) {
|
||||
t.Errorf("timeout: %v", err)
|
||||
}
|
||||
if !errors.Is(err, context.DeadlineExceeded) {
|
||||
t.Errorf("the timeout must still be recognisable: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// A request that gives up while waiting for its turn must give the turn back,
|
||||
// otherwise a burst of impatient callers makes everybody wait for nobody.
|
||||
type cancellingClock struct {
|
||||
*fakeClock
|
||||
cancel func()
|
||||
}
|
||||
|
||||
func (c *cancellingClock) Sleep(ctx context.Context, d time.Duration) error {
|
||||
c.cancel() // the caller gives up during the wait
|
||||
return ctx.Err()
|
||||
}
|
||||
|
||||
func TestAbandonedRequestsGiveTheirTurnBack(t *testing.T) {
|
||||
srv, _ := server(t, ok("{}"))
|
||||
clock := newClock()
|
||||
c := newClient(clock, func(o *Options) { o.MinInterval = time.Second })
|
||||
if _, err := c.Get(context.Background(), srv.URL); err != nil { // takes the first slot
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Ten impatient callers, each cancelled while it waits.
|
||||
for i := 0; i < 10; i++ {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
c.clock = &cancellingClock{fakeClock: clock, cancel: cancel}
|
||||
if _, err := c.Get(ctx, srv.URL); !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("caller %d: err = %v, want context.Canceled", i, err)
|
||||
}
|
||||
}
|
||||
|
||||
// The next real caller waits one interval, not eleven.
|
||||
c.clock = clock
|
||||
if _, err := c.Get(context.Background(), srv.URL); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got := clock.sleeps()
|
||||
if len(got) != 1 || got[0] != time.Second {
|
||||
t.Errorf("sleeps = %v, want a single 1s wait: the abandoned turns were not returned", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDoesNotQueueWhenTheWaitWouldPassTheDeadline(t *testing.T) {
|
||||
srv, n := server(t, ok("{}"))
|
||||
clock := newClock()
|
||||
c := newClient(clock, func(o *Options) { o.MinInterval = time.Second })
|
||||
if _, err := c.Get(context.Background(), srv.URL); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
before := n.Load()
|
||||
|
||||
// Our turn is a second away but the caller only has 100 ms.
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
|
||||
defer cancel()
|
||||
if _, err := c.Get(ctx, srv.URL); !errors.Is(err, context.DeadlineExceeded) {
|
||||
t.Fatalf("err = %v, want a deadline error without waiting", err)
|
||||
}
|
||||
if n.Load() != before || len(clock.sleeps()) != 0 {
|
||||
t.Errorf("it sent a request or slept although it could not make it in time")
|
||||
}
|
||||
|
||||
// And it took no turn: the next caller waits one interval, not two.
|
||||
if _, err := c.Get(context.Background(), srv.URL); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := clock.sleeps(); len(got) != 1 || got[0] != time.Second {
|
||||
t.Errorf("sleeps = %v, want [1s]", got)
|
||||
}
|
||||
}
|
||||
|
||||
// Requests already in flight when the provider dies all fail together. That is
|
||||
// one outage, not fifteen: the breaker must open for the first window only.
|
||||
func TestConcurrentFailuresOpenTheBreakerOnce(t *testing.T) {
|
||||
var arrived atomic.Int64
|
||||
var healthy atomic.Bool
|
||||
release := make(chan struct{})
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
if healthy.Load() {
|
||||
_, _ = w.Write([]byte("{}"))
|
||||
return
|
||||
}
|
||||
arrived.Add(1)
|
||||
<-release
|
||||
w.WriteHeader(500)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
clock := newClock()
|
||||
c := newClient(clock, func(o *Options) {
|
||||
o.Retries, o.BreakerThreshold, o.BreakerOpenFor = NoRetries, 3, 30*time.Second
|
||||
})
|
||||
|
||||
const inFlight = 12
|
||||
var wg sync.WaitGroup
|
||||
for i := 0; i < inFlight; i++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
_, _ = c.Get(context.Background(), srv.URL)
|
||||
}()
|
||||
}
|
||||
deadline := time.Now().Add(5 * time.Second)
|
||||
for arrived.Load() < inFlight && time.Now().Before(deadline) {
|
||||
time.Sleep(5 * time.Millisecond)
|
||||
}
|
||||
if arrived.Load() != inFlight {
|
||||
t.Fatalf("only %d of %d requests were in flight", arrived.Load(), inFlight)
|
||||
}
|
||||
close(release)
|
||||
wg.Wait()
|
||||
|
||||
healthy.Store(true)
|
||||
clock.advance(31 * time.Second) // just past the first window
|
||||
if _, err := c.Get(context.Background(), srv.URL); errors.Is(err, ErrCircuitOpen) {
|
||||
t.Fatal("the window grew with every failure of the same outage: it should be the first 30 s")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRedirectsAndOversizedAnswersCountAsFailures(t *testing.T) {
|
||||
for name, handler := range map[string]http.HandlerFunc{
|
||||
"redirect": func(w http.ResponseWriter, r *http.Request) {
|
||||
http.Redirect(w, r, "https://elsewhere.example/", http.StatusFound)
|
||||
},
|
||||
"too big": ok("0123456789"),
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
srv, n := server(t, handler)
|
||||
c := newClient(newClock(), func(o *Options) {
|
||||
o.Retries, o.BreakerThreshold, o.MaxBody = NoRetries, 2, 5
|
||||
})
|
||||
for i := 0; i < 2; i++ {
|
||||
if _, err := c.Get(context.Background(), srv.URL); err == nil {
|
||||
t.Fatalf("attempt %d should fail", i)
|
||||
}
|
||||
}
|
||||
before := n.Load()
|
||||
if _, err := c.Get(context.Background(), srv.URL); !errors.Is(err, ErrCircuitOpen) {
|
||||
t.Fatalf("err = %v, want ErrCircuitOpen: a provider that keeps misbehaving this way must stop being asked", err)
|
||||
}
|
||||
if n.Load() != before {
|
||||
t.Error("a request reached the provider while the breaker was open")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -165,7 +165,7 @@ func (s *Service) Update(ctx context.Context, name string, p Patch) (Info, error
|
|||
return Info{}, err
|
||||
}
|
||||
cur.APIKey = key
|
||||
if key == "" {
|
||||
if key == "" && m.needsKey {
|
||||
cur.Enabled = false // a provider that needs a key is off without one
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -259,3 +259,19 @@ func TestUserAgentIsIdentifiableAndSafe(t *testing.T) {
|
|||
}
|
||||
}
|
||||
}
|
||||
|
||||
// An empty api_key on a provider that has no keys must not switch it off.
|
||||
func TestAnEmptyKeyDoesNotTurnOffAProviderThatNeedsNone(t *testing.T) {
|
||||
s, _ := newSvc()
|
||||
ctx := context.Background()
|
||||
if _, err := s.Update(ctx, MusicBrainz, Patch{Enabled: boolp(true)}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
info, err := s.Update(ctx, MusicBrainz, Patch{Contact: strp("https://example.org/me"), APIKey: strp("")})
|
||||
if err != nil || !info.Enabled {
|
||||
t.Fatalf("%+v err=%v: musicbrainz needs no key, so an empty one must leave it on", info, err)
|
||||
}
|
||||
if _, ok, _ := s.Active(ctx, MusicBrainz); !ok {
|
||||
t.Error("musicbrainz stopped being active")
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue