feat(queue): Worker.Rate paces every send attempt
- Rate (sends per second, 0 = unlimited) builds one pacer, used right before Sender.Send - the wait comes before the send timeout starts, so waiting does not eat into it - a cancelled wait leaves the delivery queued, with no attempt recorded - tests: 5 deliveries at 2/s take 4 intervals of fake time, rate 0 never waits, retries share the pacer, cancellation while waiting, and one short real-time test (20 sends at 200/s)
This commit is contained in:
parent
b125c61fcf
commit
67ba07b642
2 changed files with 113 additions and 0 deletions
|
|
@ -60,6 +60,15 @@ type Worker struct {
|
|||
// Sign, when set, signs the final message (DKIM). On error the message is sent unsigned:
|
||||
// not delivering is worse than a missed signature.
|
||||
Sign func(msg []byte) ([]byte, error)
|
||||
// Rate is the most sends started per second, for relays that throttle (Amazon SES). Zero or
|
||||
// negative means unlimited. It is read once, at the first send.
|
||||
Rate float64
|
||||
|
||||
pacerOnce sync.Once
|
||||
pacer *pacer
|
||||
// Hooks for tests; nil means the real clock and sleep.
|
||||
pacerNow func() time.Time
|
||||
pacerSleep func(ctx context.Context, d time.Duration) error
|
||||
|
||||
wakeOnce sync.Once
|
||||
wake chan struct{}
|
||||
|
|
@ -240,6 +249,10 @@ func (w *Worker) deliver(ctx context.Context, cache *bodies, it store.OutboxItem
|
|||
if timeout <= 0 {
|
||||
timeout = defaultTimeout
|
||||
}
|
||||
// Wait for the slot before starting the timeout, so waiting does not eat into the send.
|
||||
if err := w.pace().Wait(ctx); err != nil {
|
||||
return Stats{} // shutting down: the delivery stays queued, unchanged
|
||||
}
|
||||
sctx, cancel := context.WithTimeout(ctx, timeout)
|
||||
err = w.Sender.Send(sctx, it.From, []string{it.Rcpt}, msg)
|
||||
cancel()
|
||||
|
|
@ -277,6 +290,21 @@ func (w *Worker) deliver(ctx context.Context, cache *bodies, it store.OutboxItem
|
|||
return Stats{Retried: 1}
|
||||
}
|
||||
|
||||
// pace returns the pacer in front of every send; nil (never waits) when Rate is not set.
|
||||
func (w *Worker) pace() *pacer {
|
||||
w.pacerOnce.Do(func() {
|
||||
now, sleep := w.pacerNow, w.pacerSleep
|
||||
if now == nil {
|
||||
now = time.Now
|
||||
}
|
||||
if sleep == nil {
|
||||
sleep = sleepCtx
|
||||
}
|
||||
w.pacer = newPacer(w.Rate, now, sleep)
|
||||
})
|
||||
return w.pacer
|
||||
}
|
||||
|
||||
// giveUp marks a delivery as failed for good and logs it.
|
||||
func (w *Worker) giveUp(it store.OutboxItem, reason string) Stats {
|
||||
log.Printf("queue: %s: %s", it.Rcpt, reason)
|
||||
|
|
|
|||
|
|
@ -668,3 +668,88 @@ func TestAMissingBodyIsDroppedForEveryRecipientWithOneRead(t *testing.T) {
|
|||
t.Fatalf("%d reads", n)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRateSpacesTheSendsOfAWorker(t *testing.T) {
|
||||
e := setup(t)
|
||||
e.enqueue(t, 5)
|
||||
e.w.Rate = 2 // one send every 500 ms
|
||||
f := &fakeTime{t: t0}
|
||||
e.w.pacerNow, e.w.pacerSleep = f.now, f.sleep
|
||||
st, _ := e.w.RunOnce(context.Background())
|
||||
if st.Sent != 5 {
|
||||
t.Fatalf("%+v", st)
|
||||
}
|
||||
got := f.durations() // 5 deliveries: slots at 0, 1, 2, 3 and 4 intervals
|
||||
want := []time.Duration{500 * time.Millisecond, time.Second, 1500 * time.Millisecond, 2 * time.Second}
|
||||
if len(got) != len(want) {
|
||||
t.Fatalf("sleeps %v, want %v", got, want)
|
||||
}
|
||||
for i := range want {
|
||||
if got[i] != want[i] {
|
||||
t.Fatalf("sleeps %v, want %v", got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRateOfZeroNeverWaits(t *testing.T) {
|
||||
e := setup(t)
|
||||
e.enqueue(t, 5)
|
||||
f := &fakeTime{t: t0}
|
||||
e.w.pacerNow, e.w.pacerSleep = f.now, f.sleep
|
||||
e.w.RunOnce(context.Background())
|
||||
if len(f.slept) != 0 || e.snd.count() != 5 {
|
||||
t.Fatalf("slept %v, sent %d", f.slept, e.snd.count())
|
||||
}
|
||||
}
|
||||
|
||||
func TestEverySendAttemptIsPacedEvenARetry(t *testing.T) {
|
||||
e := setup(t)
|
||||
e.enqueue(t, 1)
|
||||
e.w.Rate = 2
|
||||
f := &fakeTime{t: t0}
|
||||
e.w.pacerNow, e.w.pacerSleep = f.now, f.sleep
|
||||
e.snd.fn = func(context.Context, string, []string) error { return errors.New("try later") }
|
||||
e.w.RunOnce(context.Background())
|
||||
e.clk.set(t0.Add(time.Hour))
|
||||
e.w.RunOnce(context.Background())
|
||||
if e.snd.calls != 2 || len(f.slept) != 1 {
|
||||
t.Fatalf("calls=%d sleeps=%v: the second attempt shares the pacer with the first", e.snd.calls, f.slept)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPacingIsCancelledWithoutRecordingAnAttempt(t *testing.T) {
|
||||
e := setup(t)
|
||||
e.enqueue(t, 2)
|
||||
e.w.Rate = 1
|
||||
e.w.Concurrency = 1
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
f := &fakeTime{t: t0}
|
||||
e.w.pacerNow = f.now
|
||||
e.w.pacerSleep = func(ctx context.Context, d time.Duration) error {
|
||||
cancel() // shutdown arrives while the delivery waits for its slot
|
||||
return ctx.Err()
|
||||
}
|
||||
e.w.RunOnce(ctx)
|
||||
list, _ := e.s.ListOutbox(10)
|
||||
if len(list) != 1 || list[0].Attempts != 0 || list[0].LastError != "" {
|
||||
t.Fatalf("a shutdown while waiting is not a delivery failure: %+v", list)
|
||||
}
|
||||
if e.snd.calls != 1 {
|
||||
t.Fatalf("calls = %d: only the first delivery, which did not wait, was sent", e.snd.calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRateHoldsInRealTime(t *testing.T) {
|
||||
e := setup(t)
|
||||
e.enqueue(t, 20)
|
||||
e.w.Rate = 200 // 5 ms apart: 19 intervals take 95 ms
|
||||
start := time.Now()
|
||||
st, _ := e.w.RunOnce(context.Background())
|
||||
took := time.Since(start)
|
||||
if st.Sent != 20 {
|
||||
t.Fatalf("%+v", st)
|
||||
}
|
||||
if took < 90*time.Millisecond || took > time.Second {
|
||||
t.Fatalf("20 sends at 200/s took %v, want about 95 ms", took)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue