parsimony/internal/queue/queue_test.go
tgiachi c853ae7e61 test(queue): cover a pacer slot given up while a later caller holds its own
- a caller that gives up must not release a slot when another caller reserved after it
- a delivery that did not get its slot is not counted in the run's stats
2026-11-10 12:55:26 +01:00

758 lines
22 KiB
Go

package queue
import (
"context"
"errors"
"strings"
"sync"
"sync/atomic"
"testing"
"time"
"github.com/emersion/go-smtp"
"codeberg.org/squidd/parsimony/internal/outbound"
"codeberg.org/squidd/parsimony/internal/store"
)
var t0 = time.Unix(1_900_000_000, 0)
// script is a Sender whose behaviour the test controls.
type script struct {
mu sync.Mutex
sent []outbound.Sent
fn func(ctx context.Context, from string, to []string) error // nil: succeed
calls int32
}
func (s *script) Send(ctx context.Context, from string, to []string, data []byte) error {
atomic.AddInt32(&s.calls, 1)
if s.fn != nil {
if err := s.fn(ctx, from, to); err != nil {
return err
}
}
s.mu.Lock()
defer s.mu.Unlock()
s.sent = append(s.sent, outbound.Sent{From: from, To: append([]string(nil), to...), Data: append([]byte(nil), data...)})
return nil
}
func (s *script) count() int {
s.mu.Lock()
defer s.mu.Unlock()
return len(s.sent)
}
// clock is a test clock that the worker goroutine may read while the test moves it.
type clock struct {
mu sync.Mutex
t time.Time
}
func (c *clock) now() time.Time {
c.mu.Lock()
defer c.mu.Unlock()
return c.t
}
func (c *clock) set(t time.Time) {
c.mu.Lock()
c.t = t
c.mu.Unlock()
}
type env struct {
s *store.Store
w *Worker
snd *script
clk *clock
}
func setup(t *testing.T) *env {
t.Helper()
s, err := store.Open(":memory:")
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { s.Close() })
clk := &clock{t: t0}
s.Now = clk.now
snd := &script{}
return &env{s: s, snd: snd, clk: clk, w: &Worker{Store: s, Sender: snd}}
}
func (e *env) enqueue(t *testing.T, n int) {
t.Helper()
var items []store.NewOutbound
for i := 0; i < n; i++ {
items = append(items, store.NewOutbound{From: "bounce@x.org", Rcpt: string(rune('a'+i)) + "@x.com", Prefix: []byte("X-Rcpt: " + string(rune('a'+i)) + "\r\n")})
}
if err := e.s.Enqueue([]byte("Subject: hi\r\n\r\nbody\r\n"), items); err != nil {
t.Fatal(err)
}
}
func (e *env) pending(t *testing.T) int {
t.Helper()
st, _ := e.s.OutboxStats()
return st.Pending
}
func TestDeliversPrefixPlusMessage(t *testing.T) {
e := setup(t)
e.enqueue(t, 2)
st, err := e.w.RunOnce(context.Background())
if err != nil || st.Sent != 2 {
t.Fatalf("run: %+v %v", st, err)
}
if e.pending(t) != 0 {
t.Fatal("sent items must leave the queue")
}
// Sends run concurrently, so look recipients up instead of relying on order.
byRcpt := map[string]outbound.Sent{}
for _, m := range e.snd.sent {
byRcpt[m.To[0]] = m
}
a, b := byRcpt["a@x.com"], byRcpt["b@x.com"]
if a.From != "bounce@x.org" || len(a.To) != 1 {
t.Fatalf("envelope: %+v", a)
}
if string(a.Data) != "X-Rcpt: a\r\nSubject: hi\r\n\r\nbody\r\n" {
t.Fatalf("data = %q", a.Data)
}
if string(b.Data) != "X-Rcpt: b\r\nSubject: hi\r\n\r\nbody\r\n" {
t.Fatalf("each recipient has its own prefix: %q", b.Data)
}
}
func TestEmptyQueueIsFine(t *testing.T) {
e := setup(t)
if st, err := e.w.RunOnce(context.Background()); err != nil || st != (Stats{}) {
t.Fatalf("%+v %v", st, err)
}
}
func TestBackoffSchedule(t *testing.T) {
want := []time.Duration{time.Minute, 5 * time.Minute, 15 * time.Minute, time.Hour, 3 * time.Hour, 6 * time.Hour, 6 * time.Hour, 6 * time.Hour}
for i, w := range want {
if got := Backoff(i + 1); got != w {
t.Errorf("Backoff(%d) = %v, want %v", i+1, got, w)
}
}
if Backoff(0) != time.Minute || Backoff(-3) != time.Minute {
t.Error("nonsense input falls back to the first delay")
}
}
func TestTemporaryFailureIsRetriedLater(t *testing.T) {
e := setup(t)
e.enqueue(t, 1)
e.snd.fn = func(context.Context, string, []string) error { return errors.New("connection refused") }
st, _ := e.w.RunOnce(context.Background())
if st.Retried != 1 || st.Sent != 0 {
t.Fatalf("stats: %+v", st)
}
list, _ := e.s.ListOutbox(10)
if len(list) != 1 || list[0].Attempts != 1 || !strings.Contains(list[0].LastError, "connection refused") ||
!list[0].NextAt.Equal(t0.Add(time.Minute)) {
t.Fatalf("item: %+v", list)
}
// Not due yet: nothing is attempted.
before := atomic.LoadInt32(&e.snd.calls)
e.clk.set(t0.Add(59 * time.Second))
e.w.RunOnce(context.Background())
if atomic.LoadInt32(&e.snd.calls) != before {
t.Fatal("must wait for the backoff")
}
// Second failure waits longer.
e.clk.set(t0.Add(time.Minute))
e.w.RunOnce(context.Background())
list, _ = e.s.ListOutbox(10)
if list[0].Attempts != 2 || !list[0].NextAt.Equal(t0.Add(time.Minute+5*time.Minute)) {
t.Fatalf("second failure: %+v", list[0])
}
// The relay comes back.
e.snd.fn = nil
e.clk.set(t0.Add(10 * time.Minute))
st, _ = e.w.RunOnce(context.Background())
if st.Sent != 1 || e.pending(t) != 0 || e.snd.count() != 1 {
t.Fatalf("recovery: %+v pending=%d sent=%d", st, e.pending(t), e.snd.count())
}
}
func TestOneFailureDoesNotBlockTheOthers(t *testing.T) {
e := setup(t)
e.enqueue(t, 3)
e.snd.fn = func(_ context.Context, _ string, to []string) error {
if to[0] == "b@x.com" {
return errors.New("greylisted")
}
return nil
}
st, _ := e.w.RunOnce(context.Background())
if st.Sent != 2 || st.Retried != 1 || e.pending(t) != 1 {
t.Fatalf("%+v pending=%d", st, e.pending(t))
}
list, _ := e.s.ListOutbox(10)
if list[0].Rcpt != "b@x.com" {
t.Fatalf("the failed recipient stays queued, got %q", list[0].Rcpt)
}
}
func TestPermanentFailureIsKeptVisibleAndNeverRetried(t *testing.T) {
e := setup(t)
e.enqueue(t, 1)
calls := 0
e.snd.fn = func(context.Context, string, []string) error {
calls++
return &smtp.SMTPError{Code: 554, EnhancedCode: smtp.EnhancedCode{5, 7, 1}, Message: "relay access denied"}
}
st, _ := e.w.RunOnce(context.Background())
if st.Failed != 1 || st.Dropped != 0 || st.Bounced != 0 {
t.Fatalf("%+v", st)
}
stats, _ := e.s.OutboxStats()
if stats.Pending != 0 || stats.Failed != 1 {
t.Fatalf("the delivery is not lost, it is listed as failed: %+v", stats)
}
list, _ := e.s.ListOutbox(10)
if len(list) != 1 || !list[0].Failed || !strings.Contains(list[0].LastError, "relay access denied") {
t.Fatalf("the admin sees why: %+v", list)
}
e.clk.set(t0.Add(48 * time.Hour))
e.w.RunOnce(context.Background())
if calls != 1 {
t.Fatalf("a refused delivery is never retried, relay called %d times", calls)
}
}
func TestFailedDeliveriesAreForgottenAfterAWeekByTheWorker(t *testing.T) {
e := setup(t)
e.enqueue(t, 1)
e.snd.fn = func(context.Context, string, []string) error {
return &smtp.SMTPError{Code: 554, EnhancedCode: smtp.EnhancedCode{5, 7, 1}, Message: "no"}
}
e.w.RunOnce(context.Background())
e.clk.set(t0.Add(8 * 24 * time.Hour))
e.w.RunOnce(context.Background())
if stats, _ := e.s.OutboxStats(); stats.Failed != 0 {
t.Fatalf("an old failure is purged on the next run: %+v", stats)
}
}
func TestBadMailboxCountsAHardBounceAndDisablesAfterThree(t *testing.T) {
e := setup(t)
l, _ := e.s.CreateList("dev", "example.org", "Dev")
e.s.AddMember(l.ID, "gone@x.com", "")
ms, _ := e.s.Members(l.ID)
id := ms[0].ID
e.snd.fn = func(context.Context, string, []string) error {
return outbound.AtRcpt(&smtp.SMTPError{Code: 550, EnhancedCode: smtp.EnhancedCode{5, 1, 1}, Message: "no such user"})
}
for i := 1; i <= 3; i++ {
e.s.Enqueue([]byte("Subject: x\r\n\r\nx"), []store.NewOutbound{{From: "b@x.org", Rcpt: "gone@x.com", MemberID: id}})
st, _ := e.w.RunOnce(context.Background())
if st.Dropped != 1 || st.Bounced != 1 {
t.Fatalf("round %d: %+v", i, st)
}
m, _ := e.s.MemberByID(id)
if m.BounceCount != i || m.Disabled != (i == 3) {
t.Fatalf("round %d: member %+v", i, m)
}
}
if e.pending(t) != 0 {
t.Fatal("permanent failures leave the queue")
}
}
func TestOtherPermanentRejectionsNeverCountAgainstTheMember(t *testing.T) {
// A misconfigured relay that refuses everything must not empty the list.
e := setup(t)
l, _ := e.s.CreateList("dev", "example.org", "Dev")
e.s.AddMember(l.ID, "ok@x.com", "")
ms, _ := e.s.Members(l.ID)
e.snd.fn = func(context.Context, string, []string) error {
return &smtp.SMTPError{Code: 554, EnhancedCode: smtp.EnhancedCode{5, 7, 1}, Message: "relay access denied"}
}
for i := 0; i < 5; i++ {
e.s.Enqueue([]byte("x"), []store.NewOutbound{{From: "b@x.org", Rcpt: "ok@x.com", MemberID: ms[0].ID}})
st, _ := e.w.RunOnce(context.Background())
if st.Failed != 1 || st.Bounced != 0 {
t.Fatalf("round %d: %+v", i, st)
}
}
if m, _ := e.s.MemberByID(ms[0].ID); m.BounceCount != 0 || m.Disabled {
t.Fatalf("member was punished for a relay problem: %+v", m)
}
}
func TestBadMailboxForSystemMailOrUnknownMemberIsHarmless(t *testing.T) {
e := setup(t)
e.snd.fn = func(context.Context, string, []string) error {
return outbound.AtRcpt(&smtp.SMTPError{Code: 550, EnhancedCode: smtp.EnhancedCode{5, 1, 1}, Message: "no such user"})
}
e.s.Enqueue([]byte("x"), []store.NewOutbound{
{From: "b@x.org", Rcpt: "a@x.com", MemberID: 0}, // confirmation mail
{From: "b@x.org", Rcpt: "b@x.com", MemberID: 404}, // member was deleted meanwhile
})
st, err := e.w.RunOnce(context.Background())
if err != nil || st.Dropped != 2 || e.pending(t) != 0 {
t.Fatalf("%+v %v pending=%d", st, err, e.pending(t))
}
}
func TestGivesUpAfterMaxAge(t *testing.T) {
e := setup(t)
e.enqueue(t, 1)
e.snd.fn = func(context.Context, string, []string) error { return errors.New("still down") }
e.clk.set(t0.Add(MaxAge - time.Minute))
if st, _ := e.w.RunOnce(context.Background()); st.Retried != 1 {
t.Fatalf("just inside the limit is retried: %+v", st)
}
e.clk.set(t0.Add(MaxAge + time.Minute))
// The retry delay may not have passed yet; move far enough.
e.clk.set(e.clk.now().Add(7 * time.Hour))
st, _ := e.w.RunOnce(context.Background())
if st.Failed != 1 || e.pending(t) != 0 {
t.Fatalf("too old: %+v pending=%d", st, e.pending(t))
}
list, _ := e.s.ListOutbox(10)
if len(list) != 1 || !list[0].Failed || !strings.Contains(list[0].LastError, "gave up") || !strings.Contains(list[0].LastError, "still down") {
t.Fatalf("it stays listed with the reason: %+v", list)
}
}
func TestOldMailIsStillDeliveredIfTheRelayWorks(t *testing.T) {
e := setup(t)
e.enqueue(t, 1)
e.clk.set(t0.Add(MaxAge * 2))
if st, _ := e.w.RunOnce(context.Background()); st.Sent != 1 {
t.Fatalf("age only matters for failures: %+v", st)
}
}
func TestConcurrencyIsBounded(t *testing.T) {
e := setup(t)
e.enqueue(t, 10)
e.w.Concurrency = 3
var cur, max int32
e.snd.fn = func(ctx context.Context, _ string, _ []string) error {
n := atomic.AddInt32(&cur, 1)
for {
m := atomic.LoadInt32(&max)
if n <= m || atomic.CompareAndSwapInt32(&max, m, n) {
break
}
}
time.Sleep(20 * time.Millisecond)
atomic.AddInt32(&cur, -1)
return nil
}
st, _ := e.w.RunOnce(context.Background())
if st.Sent != 10 {
t.Fatalf("%+v", st)
}
if max != 3 {
t.Fatalf("max concurrent sends = %d, want 3", max)
}
}
func TestBatchLimitsOneRun(t *testing.T) {
e := setup(t)
e.enqueue(t, 5)
e.w.Batch = 2
st, _ := e.w.RunOnce(context.Background())
if st.Sent != 2 || e.pending(t) != 3 {
t.Fatalf("%+v pending=%d", st, e.pending(t))
}
}
func TestInterruptedSendStaysQueuedUntouched(t *testing.T) {
e := setup(t)
e.enqueue(t, 1)
ctx, cancel := context.WithCancel(context.Background())
e.snd.fn = func(ctx context.Context, _ string, _ []string) error {
cancel() // shutdown arrives while the send is in flight
<-ctx.Done()
return ctx.Err()
}
e.w.RunOnce(ctx)
list, _ := e.s.ListOutbox(10)
if len(list) != 1 || list[0].Attempts != 0 || list[0].LastError != "" {
t.Fatalf("a shutdown is not a delivery failure: %+v", list)
}
}
func TestSlowRelayTimesOutAndIsRetried(t *testing.T) {
e := setup(t)
e.enqueue(t, 1)
e.w.Timeout = 30 * time.Millisecond
e.snd.fn = func(ctx context.Context, _ string, _ []string) error {
<-ctx.Done()
return ctx.Err()
}
st, _ := e.w.RunOnce(context.Background())
if st.Retried != 1 {
t.Fatalf("a hung relay is a temporary failure: %+v", st)
}
list, _ := e.s.ListOutbox(10)
if !strings.Contains(list[0].LastError, "deadline") {
t.Fatalf("last error: %q", list[0].LastError)
}
}
func waitFor(t *testing.T, what string, cond func() bool) {
t.Helper()
deadline := time.Now().Add(3 * time.Second)
for !cond() {
if time.Now().After(deadline) {
t.Fatalf("timed out waiting for %s", what)
}
time.Sleep(5 * time.Millisecond)
}
}
func TestRunDrainsTheQueueInBatchesAndStopsOnCancel(t *testing.T) {
e := setup(t)
e.enqueue(t, 7)
e.w.Batch = 2
e.w.Interval = time.Hour
ctx, cancel := context.WithCancel(context.Background())
done := make(chan struct{})
go func() { e.w.Run(ctx); close(done) }()
waitFor(t, "7 deliveries", func() bool { return e.snd.count() == 7 })
cancel()
select {
case <-done:
case <-time.After(2 * time.Second):
t.Fatal("Run did not stop")
}
}
func TestWakeTriggersAnImmediateRun(t *testing.T) {
e := setup(t)
e.w.Interval = time.Hour
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
go e.w.Run(ctx)
time.Sleep(50 * time.Millisecond) // the first run finds nothing and goes idle
e.enqueue(t, 1)
e.w.Wake()
waitFor(t, "woken delivery", func() bool { return e.snd.count() == 1 })
}
func TestWakeBeforeRunAndRepeatedWakesDoNotBlock(t *testing.T) {
e := setup(t)
done := make(chan struct{})
go func() {
for i := 0; i < 100; i++ {
e.w.Wake()
}
close(done)
}()
select {
case <-done:
case <-time.After(time.Second):
t.Fatal("Wake blocked")
}
}
func TestIdleRunPollsForDueRetries(t *testing.T) {
e := setup(t)
e.enqueue(t, 1)
fail := int32(1)
e.snd.fn = func(context.Context, string, []string) error {
if atomic.LoadInt32(&fail) == 1 {
return errors.New("down")
}
return nil
}
e.w.Interval = 20 * time.Millisecond
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
go e.w.Run(ctx)
waitFor(t, "first failed attempt", func() bool { st, _ := e.s.OutboxStats(); return st.Retrying == 1 })
atomic.StoreInt32(&fail, 0)
e.clk.set(t0.Add(2 * time.Minute)) // the backoff has passed; no Wake needed
waitFor(t, "retry by polling", func() bool { return e.snd.count() == 1 })
}
func TestSignHookSeesTheFinalMessageIncludingThePerRecipientHeaders(t *testing.T) {
e := setup(t)
e.enqueue(t, 1)
var seen []byte
e.w.Sign = func(msg []byte) ([]byte, error) {
seen = append([]byte(nil), msg...)
return append([]byte("X-Signed: yes\r\n"), msg...), nil
}
if st, _ := e.w.RunOnce(context.Background()); st.Sent != 1 {
t.Fatalf("%+v", st)
}
if string(seen) != "X-Rcpt: a\r\nSubject: hi\r\n\r\nbody\r\n" {
t.Fatalf("the signer must see prefix + message, so List-Unsubscribe is covered: %q", seen)
}
if string(e.snd.sent[0].Data) != "X-Signed: yes\r\nX-Rcpt: a\r\nSubject: hi\r\n\r\nbody\r\n" {
t.Fatalf("the relay must get the signed message: %q", e.snd.sent[0].Data)
}
}
func TestSigningFailureSendsTheMessageUnsigned(t *testing.T) {
e := setup(t)
e.enqueue(t, 1)
e.w.Sign = func(msg []byte) ([]byte, error) { return nil, errors.New("hsm unavailable") }
st, _ := e.w.RunOnce(context.Background())
if st.Sent != 1 || e.pending(t) != 0 {
t.Fatalf("a signing error must not stop delivery: %+v", st)
}
if string(e.snd.sent[0].Data) != "X-Rcpt: a\r\nSubject: hi\r\n\r\nbody\r\n" {
t.Fatalf("unsigned message expected: %q", e.snd.sent[0].Data)
}
}
func TestEachAttemptIsSignedAgain(t *testing.T) {
e := setup(t)
e.enqueue(t, 1)
calls := 0
e.w.Sign = func(msg []byte) ([]byte, error) { calls++; return msg, nil }
failures := int32(1)
e.snd.fn = func(context.Context, string, []string) error {
if atomic.AddInt32(&failures, -1) >= 0 {
return errors.New("down")
}
return nil
}
e.w.RunOnce(context.Background())
e.clk.set(t0.Add(2 * time.Minute))
e.w.RunOnce(context.Background())
if calls != 2 {
t.Fatalf("a retry is signed afresh (new timestamp), signing calls = %d", calls)
}
}
func TestARefusedSenderIsNotAMissingMailbox(t *testing.T) {
// A relay that rejects the list's own sender address (5.1.8, 5.1.7, 5.1.0 or even 5.1.1 answering
// MAIL FROM) says nothing about the recipients: the members must not be struck, and the mail must
// stay visible as failed instead of vanishing.
e := setup(t)
l, _ := e.s.CreateList("dev", "example.org", "Dev")
e.s.AddMember(l.ID, "ok@x.com", "")
ms, _ := e.s.Members(l.ID)
e.snd.fn = func(context.Context, string, []string) error {
return &outbound.PhaseError{Phase: "MAIL", Err: &smtp.SMTPError{Code: 550, EnhancedCode: smtp.EnhancedCode{5, 1, 8}, Message: "Bad outbound sender"}}
}
for i := 0; i < 5; i++ {
e.s.Enqueue([]byte("x"), []store.NewOutbound{{From: "dev+bounce-1-x@example.org", Rcpt: "ok@x.com", MemberID: ms[0].ID}})
st, _ := e.w.RunOnce(context.Background())
if st.Failed != 1 || st.Bounced != 0 || st.Dropped != 0 {
t.Fatalf("round %d: %+v", i+1, st)
}
}
if m, _ := e.s.MemberByID(ms[0].ID); m.BounceCount != 0 || m.Disabled {
t.Fatalf("the member was struck for our own sender problem: %+v", m)
}
if stats, _ := e.s.OutboxStats(); stats.Failed != 5 {
t.Fatalf("every delivery stays listed as failed: %+v", stats)
}
}
func TestPurgeRunsOncePerIntervalNotOnEveryBatch(t *testing.T) {
e := setup(t)
var purges int32
e.w.purge = func() (int, error) { atomic.AddInt32(&purges, 1); return 0, nil }
e.w.Batch = 1
e.enqueue(t, 3)
for i := 0; i < 3; i++ {
if st, err := e.w.RunOnce(context.Background()); err != nil || st.Sent != 1 {
t.Fatalf("run %d: %+v %v", i, st, err)
}
}
if n := atomic.LoadInt32(&purges); n != 1 {
t.Fatalf("purge ran %d times in 3 batches, want 1", n)
}
e.clk.set(t0.Add(purgeEvery - time.Second))
e.w.RunOnce(context.Background())
if n := atomic.LoadInt32(&purges); n != 1 {
t.Fatalf("purge ran again before the interval: %d", n)
}
e.clk.set(t0.Add(purgeEvery))
e.w.RunOnce(context.Background())
if n := atomic.LoadInt32(&purges); n != 2 {
t.Fatalf("purge must run again after the interval: %d", n)
}
e.w.RunOnce(context.Background())
if n := atomic.LoadInt32(&purges); n != 2 {
t.Fatalf("interval counts from the last purge: %d", n)
}
}
func TestFailedPurgeIsRetriedOnTheNextRun(t *testing.T) {
e := setup(t)
var purges int32
e.w.purge = func() (int, error) { atomic.AddInt32(&purges, 1); return 0, errors.New("db busy") }
e.w.RunOnce(context.Background())
e.w.RunOnce(context.Background())
if n := atomic.LoadInt32(&purges); n != 2 {
t.Fatalf("a failed purge must not delay the next attempt: %d runs", n)
}
}
func TestOneBodyReadServesEveryRecipientOfAMessage(t *testing.T) {
e := setup(t)
reads := map[int64]int{}
var mu sync.Mutex
e.w.readBody = func(id int64) ([]byte, error) {
mu.Lock()
reads[id]++
mu.Unlock()
return e.s.OutboxData(id)
}
e.enqueue(t, 6)
e.w.Concurrency = 6
if st, err := e.w.RunOnce(context.Background()); err != nil || st.Sent != 6 {
t.Fatalf("run: %+v %v", st, err)
}
if len(reads) != 1 {
t.Fatalf("reads = %v, want one message", reads)
}
for id, n := range reads {
if n != 1 {
t.Fatalf("message %d read %d times, want 1", id, n)
}
}
for _, m := range e.snd.sent {
if !strings.HasSuffix(string(m.Data), "Subject: hi\r\n\r\nbody\r\n") {
t.Fatalf("data = %q", m.Data)
}
}
// The cache lives for one run only: a later run reads again.
e.enqueue(t, 1)
e.w.RunOnce(context.Background())
total := 0
for _, n := range reads {
total += n
}
if total != 2 {
t.Fatalf("the next run must read its own body: %v", reads)
}
}
func TestEachDistinctMessageIsReadOnce(t *testing.T) {
e := setup(t)
var calls int32
e.w.readBody = func(id int64) ([]byte, error) { atomic.AddInt32(&calls, 1); return e.s.OutboxData(id) }
for i := 0; i < 3; i++ {
e.enqueue(t, 2)
}
if st, _ := e.w.RunOnce(context.Background()); st.Sent != 6 {
t.Fatalf("sent %d", st.Sent)
}
if n := atomic.LoadInt32(&calls); n != 3 {
t.Fatalf("%d reads for 3 messages", n)
}
}
func TestAMissingBodyIsDroppedForEveryRecipientWithOneRead(t *testing.T) {
e := setup(t)
var calls int32
e.w.readBody = func(int64) ([]byte, error) { atomic.AddInt32(&calls, 1); return nil, store.ErrNotFound }
e.enqueue(t, 3)
st, _ := e.w.RunOnce(context.Background())
if st.Dropped != 3 || e.snd.count() != 0 {
t.Fatalf("stats %+v", st)
}
if n := atomic.LoadInt32(&calls); n != 1 {
t.Fatalf("%d reads", n)
}
}
func TestRateSpacesTheSendsOfAWorker(t *testing.T) {
e := setup(t)
e.enqueue(t, 5)
e.w.Rate = 2 // one send every 500 ms
f := &fakeTime{t: t0}
e.w.pacerNow, e.w.pacerSleep = f.now, f.sleep
st, _ := e.w.RunOnce(context.Background())
if st.Sent != 5 {
t.Fatalf("%+v", st)
}
got := f.durations() // 5 deliveries: slots at 0, 1, 2, 3 and 4 intervals
want := []time.Duration{500 * time.Millisecond, time.Second, 1500 * time.Millisecond, 2 * time.Second}
if len(got) != len(want) {
t.Fatalf("sleeps %v, want %v", got, want)
}
for i := range want {
if got[i] != want[i] {
t.Fatalf("sleeps %v, want %v", got, want)
}
}
}
func TestRateOfZeroNeverWaits(t *testing.T) {
e := setup(t)
e.enqueue(t, 5)
f := &fakeTime{t: t0}
e.w.pacerNow, e.w.pacerSleep = f.now, f.sleep
e.w.RunOnce(context.Background())
if len(f.slept) != 0 || e.snd.count() != 5 {
t.Fatalf("slept %v, sent %d", f.slept, e.snd.count())
}
}
func TestEverySendAttemptIsPacedEvenARetry(t *testing.T) {
e := setup(t)
e.enqueue(t, 1)
e.w.Rate = 2
f := &fakeTime{t: t0}
e.w.pacerNow, e.w.pacerSleep = f.now, f.sleep
e.snd.fn = func(context.Context, string, []string) error { return errors.New("try later") }
e.w.RunOnce(context.Background())
e.clk.set(t0.Add(time.Hour))
e.w.RunOnce(context.Background())
if e.snd.calls != 2 || len(f.slept) != 1 {
t.Fatalf("calls=%d sleeps=%v: the second attempt shares the pacer with the first", e.snd.calls, f.slept)
}
}
func TestPacingIsCancelledWithoutRecordingAnAttempt(t *testing.T) {
e := setup(t)
e.enqueue(t, 2)
e.w.Rate = 1
e.w.Concurrency = 1
ctx, cancel := context.WithCancel(context.Background())
f := &fakeTime{t: t0}
e.w.pacerNow = f.now
e.w.pacerSleep = func(ctx context.Context, d time.Duration) error {
cancel() // shutdown arrives while the delivery waits for its slot
return ctx.Err()
}
st, _ := e.w.RunOnce(ctx)
if st != (Stats{Sent: 1}) {
t.Fatalf("stats %+v: the delivery that did not get its slot is not counted", st)
}
list, _ := e.s.ListOutbox(10)
if len(list) != 1 || list[0].Attempts != 0 || list[0].LastError != "" {
t.Fatalf("a shutdown while waiting is not a delivery failure: %+v", list)
}
if e.snd.calls != 1 {
t.Fatalf("calls = %d: only the first delivery, which did not wait, was sent", e.snd.calls)
}
}
func TestRateHoldsInRealTime(t *testing.T) {
e := setup(t)
e.enqueue(t, 20)
e.w.Rate = 200 // 5 ms apart: 19 intervals take 95 ms
start := time.Now()
st, _ := e.w.RunOnce(context.Background())
took := time.Since(start)
if st.Sent != 20 {
t.Fatalf("%+v", st)
}
if took < 90*time.Millisecond || took > time.Second {
t.Fatalf("20 sends at 200/s took %v, want about 95 ms", took)
}
}