package queue import ( "context" "errors" "strings" "sync" "sync/atomic" "testing" "time" "github.com/emersion/go-smtp" "codeberg.org/squidd/parsimony/internal/outbound" "codeberg.org/squidd/parsimony/internal/store" ) var t0 = time.Unix(1_900_000_000, 0) // script is a Sender whose behaviour the test controls. type script struct { mu sync.Mutex sent []outbound.Sent fn func(ctx context.Context, from string, to []string) error // nil: succeed calls int32 } func (s *script) Send(ctx context.Context, from string, to []string, data []byte) error { atomic.AddInt32(&s.calls, 1) if s.fn != nil { if err := s.fn(ctx, from, to); err != nil { return err } } s.mu.Lock() defer s.mu.Unlock() s.sent = append(s.sent, outbound.Sent{From: from, To: append([]string(nil), to...), Data: append([]byte(nil), data...)}) return nil } func (s *script) count() int { s.mu.Lock() defer s.mu.Unlock() return len(s.sent) } // clock is a test clock that the worker goroutine may read while the test moves it. type clock struct { mu sync.Mutex t time.Time } func (c *clock) now() time.Time { c.mu.Lock() defer c.mu.Unlock() return c.t } func (c *clock) set(t time.Time) { c.mu.Lock() c.t = t c.mu.Unlock() } type env struct { s *store.Store w *Worker snd *script clk *clock } func setup(t *testing.T) *env { t.Helper() s, err := store.Open(":memory:") if err != nil { t.Fatal(err) } t.Cleanup(func() { s.Close() }) clk := &clock{t: t0} s.Now = clk.now snd := &script{} return &env{s: s, snd: snd, clk: clk, w: &Worker{Store: s, Sender: snd}} } func (e *env) enqueue(t *testing.T, n int) { t.Helper() var items []store.NewOutbound for i := 0; i < n; i++ { items = append(items, store.NewOutbound{From: "bounce@x.org", Rcpt: string(rune('a'+i)) + "@x.com", Prefix: []byte("X-Rcpt: " + string(rune('a'+i)) + "\r\n")}) } if err := e.s.Enqueue([]byte("Subject: hi\r\n\r\nbody\r\n"), items); err != nil { t.Fatal(err) } } func (e *env) pending(t *testing.T) int { t.Helper() st, _ := e.s.OutboxStats() return st.Pending } func TestDeliversPrefixPlusMessage(t *testing.T) { e := setup(t) e.enqueue(t, 2) st, err := e.w.RunOnce(context.Background()) if err != nil || st.Sent != 2 { t.Fatalf("run: %+v %v", st, err) } if e.pending(t) != 0 { t.Fatal("sent items must leave the queue") } // Sends run concurrently, so look recipients up instead of relying on order. byRcpt := map[string]outbound.Sent{} for _, m := range e.snd.sent { byRcpt[m.To[0]] = m } a, b := byRcpt["a@x.com"], byRcpt["b@x.com"] if a.From != "bounce@x.org" || len(a.To) != 1 { t.Fatalf("envelope: %+v", a) } if string(a.Data) != "X-Rcpt: a\r\nSubject: hi\r\n\r\nbody\r\n" { t.Fatalf("data = %q", a.Data) } if string(b.Data) != "X-Rcpt: b\r\nSubject: hi\r\n\r\nbody\r\n" { t.Fatalf("each recipient has its own prefix: %q", b.Data) } } func TestEmptyQueueIsFine(t *testing.T) { e := setup(t) if st, err := e.w.RunOnce(context.Background()); err != nil || st != (Stats{}) { t.Fatalf("%+v %v", st, err) } } func TestBackoffSchedule(t *testing.T) { want := []time.Duration{time.Minute, 5 * time.Minute, 15 * time.Minute, time.Hour, 3 * time.Hour, 6 * time.Hour, 6 * time.Hour, 6 * time.Hour} for i, w := range want { if got := Backoff(i + 1); got != w { t.Errorf("Backoff(%d) = %v, want %v", i+1, got, w) } } if Backoff(0) != time.Minute || Backoff(-3) != time.Minute { t.Error("nonsense input falls back to the first delay") } } func TestTemporaryFailureIsRetriedLater(t *testing.T) { e := setup(t) e.enqueue(t, 1) e.snd.fn = func(context.Context, string, []string) error { return errors.New("connection refused") } st, _ := e.w.RunOnce(context.Background()) if st.Retried != 1 || st.Sent != 0 { t.Fatalf("stats: %+v", st) } list, _ := e.s.ListOutbox(10) if len(list) != 1 || list[0].Attempts != 1 || !strings.Contains(list[0].LastError, "connection refused") || !list[0].NextAt.Equal(t0.Add(time.Minute)) { t.Fatalf("item: %+v", list) } // Not due yet: nothing is attempted. before := atomic.LoadInt32(&e.snd.calls) e.clk.set(t0.Add(59 * time.Second)) e.w.RunOnce(context.Background()) if atomic.LoadInt32(&e.snd.calls) != before { t.Fatal("must wait for the backoff") } // Second failure waits longer. e.clk.set(t0.Add(time.Minute)) e.w.RunOnce(context.Background()) list, _ = e.s.ListOutbox(10) if list[0].Attempts != 2 || !list[0].NextAt.Equal(t0.Add(time.Minute+5*time.Minute)) { t.Fatalf("second failure: %+v", list[0]) } // The relay comes back. e.snd.fn = nil e.clk.set(t0.Add(10 * time.Minute)) st, _ = e.w.RunOnce(context.Background()) if st.Sent != 1 || e.pending(t) != 0 || e.snd.count() != 1 { t.Fatalf("recovery: %+v pending=%d sent=%d", st, e.pending(t), e.snd.count()) } } func TestOneFailureDoesNotBlockTheOthers(t *testing.T) { e := setup(t) e.enqueue(t, 3) e.snd.fn = func(_ context.Context, _ string, to []string) error { if to[0] == "b@x.com" { return errors.New("greylisted") } return nil } st, _ := e.w.RunOnce(context.Background()) if st.Sent != 2 || st.Retried != 1 || e.pending(t) != 1 { t.Fatalf("%+v pending=%d", st, e.pending(t)) } list, _ := e.s.ListOutbox(10) if list[0].Rcpt != "b@x.com" { t.Fatalf("the failed recipient stays queued, got %q", list[0].Rcpt) } } func TestPermanentFailureIsKeptVisibleAndNeverRetried(t *testing.T) { e := setup(t) e.enqueue(t, 1) calls := 0 e.snd.fn = func(context.Context, string, []string) error { calls++ return &smtp.SMTPError{Code: 554, EnhancedCode: smtp.EnhancedCode{5, 7, 1}, Message: "relay access denied"} } st, _ := e.w.RunOnce(context.Background()) if st.Failed != 1 || st.Dropped != 0 || st.Bounced != 0 { t.Fatalf("%+v", st) } stats, _ := e.s.OutboxStats() if stats.Pending != 0 || stats.Failed != 1 { t.Fatalf("the delivery is not lost, it is listed as failed: %+v", stats) } list, _ := e.s.ListOutbox(10) if len(list) != 1 || !list[0].Failed || !strings.Contains(list[0].LastError, "relay access denied") { t.Fatalf("the admin sees why: %+v", list) } e.clk.set(t0.Add(48 * time.Hour)) e.w.RunOnce(context.Background()) if calls != 1 { t.Fatalf("a refused delivery is never retried, relay called %d times", calls) } } func TestFailedDeliveriesAreForgottenAfterAWeekByTheWorker(t *testing.T) { e := setup(t) e.enqueue(t, 1) e.snd.fn = func(context.Context, string, []string) error { return &smtp.SMTPError{Code: 554, EnhancedCode: smtp.EnhancedCode{5, 7, 1}, Message: "no"} } e.w.RunOnce(context.Background()) e.clk.set(t0.Add(8 * 24 * time.Hour)) e.w.RunOnce(context.Background()) if stats, _ := e.s.OutboxStats(); stats.Failed != 0 { t.Fatalf("an old failure is purged on the next run: %+v", stats) } } func TestBadMailboxCountsAHardBounceAndDisablesAfterThree(t *testing.T) { e := setup(t) l, _ := e.s.CreateList("dev", "example.org", "Dev") e.s.AddMember(l.ID, "gone@x.com", "") ms, _ := e.s.Members(l.ID) id := ms[0].ID e.snd.fn = func(context.Context, string, []string) error { return outbound.AtRcpt(&smtp.SMTPError{Code: 550, EnhancedCode: smtp.EnhancedCode{5, 1, 1}, Message: "no such user"}) } for i := 1; i <= 3; i++ { e.s.Enqueue([]byte("Subject: x\r\n\r\nx"), []store.NewOutbound{{From: "b@x.org", Rcpt: "gone@x.com", MemberID: id}}) st, _ := e.w.RunOnce(context.Background()) if st.Dropped != 1 || st.Bounced != 1 { t.Fatalf("round %d: %+v", i, st) } m, _ := e.s.MemberByID(id) if m.BounceCount != i || m.Disabled != (i == 3) { t.Fatalf("round %d: member %+v", i, m) } } if e.pending(t) != 0 { t.Fatal("permanent failures leave the queue") } } func TestOtherPermanentRejectionsNeverCountAgainstTheMember(t *testing.T) { // A misconfigured relay that refuses everything must not empty the list. e := setup(t) l, _ := e.s.CreateList("dev", "example.org", "Dev") e.s.AddMember(l.ID, "ok@x.com", "") ms, _ := e.s.Members(l.ID) e.snd.fn = func(context.Context, string, []string) error { return &smtp.SMTPError{Code: 554, EnhancedCode: smtp.EnhancedCode{5, 7, 1}, Message: "relay access denied"} } for i := 0; i < 5; i++ { e.s.Enqueue([]byte("x"), []store.NewOutbound{{From: "b@x.org", Rcpt: "ok@x.com", MemberID: ms[0].ID}}) st, _ := e.w.RunOnce(context.Background()) if st.Failed != 1 || st.Bounced != 0 { t.Fatalf("round %d: %+v", i, st) } } if m, _ := e.s.MemberByID(ms[0].ID); m.BounceCount != 0 || m.Disabled { t.Fatalf("member was punished for a relay problem: %+v", m) } } func TestBadMailboxForSystemMailOrUnknownMemberIsHarmless(t *testing.T) { e := setup(t) e.snd.fn = func(context.Context, string, []string) error { return outbound.AtRcpt(&smtp.SMTPError{Code: 550, EnhancedCode: smtp.EnhancedCode{5, 1, 1}, Message: "no such user"}) } e.s.Enqueue([]byte("x"), []store.NewOutbound{ {From: "b@x.org", Rcpt: "a@x.com", MemberID: 0}, // confirmation mail {From: "b@x.org", Rcpt: "b@x.com", MemberID: 404}, // member was deleted meanwhile }) st, err := e.w.RunOnce(context.Background()) if err != nil || st.Dropped != 2 || e.pending(t) != 0 { t.Fatalf("%+v %v pending=%d", st, err, e.pending(t)) } } func TestGivesUpAfterMaxAge(t *testing.T) { e := setup(t) e.enqueue(t, 1) e.snd.fn = func(context.Context, string, []string) error { return errors.New("still down") } e.clk.set(t0.Add(MaxAge - time.Minute)) if st, _ := e.w.RunOnce(context.Background()); st.Retried != 1 { t.Fatalf("just inside the limit is retried: %+v", st) } e.clk.set(t0.Add(MaxAge + time.Minute)) // The retry delay may not have passed yet; move far enough. e.clk.set(e.clk.now().Add(7 * time.Hour)) st, _ := e.w.RunOnce(context.Background()) if st.Failed != 1 || e.pending(t) != 0 { t.Fatalf("too old: %+v pending=%d", st, e.pending(t)) } list, _ := e.s.ListOutbox(10) if len(list) != 1 || !list[0].Failed || !strings.Contains(list[0].LastError, "gave up") || !strings.Contains(list[0].LastError, "still down") { t.Fatalf("it stays listed with the reason: %+v", list) } } func TestOldMailIsStillDeliveredIfTheRelayWorks(t *testing.T) { e := setup(t) e.enqueue(t, 1) e.clk.set(t0.Add(MaxAge * 2)) if st, _ := e.w.RunOnce(context.Background()); st.Sent != 1 { t.Fatalf("age only matters for failures: %+v", st) } } func TestConcurrencyIsBounded(t *testing.T) { e := setup(t) e.enqueue(t, 10) e.w.Concurrency = 3 var cur, max int32 e.snd.fn = func(ctx context.Context, _ string, _ []string) error { n := atomic.AddInt32(&cur, 1) for { m := atomic.LoadInt32(&max) if n <= m || atomic.CompareAndSwapInt32(&max, m, n) { break } } time.Sleep(20 * time.Millisecond) atomic.AddInt32(&cur, -1) return nil } st, _ := e.w.RunOnce(context.Background()) if st.Sent != 10 { t.Fatalf("%+v", st) } if max != 3 { t.Fatalf("max concurrent sends = %d, want 3", max) } } func TestBatchLimitsOneRun(t *testing.T) { e := setup(t) e.enqueue(t, 5) e.w.Batch = 2 st, _ := e.w.RunOnce(context.Background()) if st.Sent != 2 || e.pending(t) != 3 { t.Fatalf("%+v pending=%d", st, e.pending(t)) } } func TestInterruptedSendStaysQueuedUntouched(t *testing.T) { e := setup(t) e.enqueue(t, 1) ctx, cancel := context.WithCancel(context.Background()) e.snd.fn = func(ctx context.Context, _ string, _ []string) error { cancel() // shutdown arrives while the send is in flight <-ctx.Done() return ctx.Err() } e.w.RunOnce(ctx) list, _ := e.s.ListOutbox(10) if len(list) != 1 || list[0].Attempts != 0 || list[0].LastError != "" { t.Fatalf("a shutdown is not a delivery failure: %+v", list) } } func TestSlowRelayTimesOutAndIsRetried(t *testing.T) { e := setup(t) e.enqueue(t, 1) e.w.Timeout = 30 * time.Millisecond e.snd.fn = func(ctx context.Context, _ string, _ []string) error { <-ctx.Done() return ctx.Err() } st, _ := e.w.RunOnce(context.Background()) if st.Retried != 1 { t.Fatalf("a hung relay is a temporary failure: %+v", st) } list, _ := e.s.ListOutbox(10) if !strings.Contains(list[0].LastError, "deadline") { t.Fatalf("last error: %q", list[0].LastError) } } func TestTheQueueLeavesTheSendDeadlinesToTheSender(t *testing.T) { // One total deadline cut a relay that took over a minute to answer the final dot: the message // was accepted, then retried and delivered again. The sender bounds each SMTP phase itself. e := setup(t) e.enqueue(t, 1) var hasDeadline bool e.snd.fn = func(ctx context.Context, _ string, _ []string) error { _, hasDeadline = ctx.Deadline() return nil } if st, _ := e.w.RunOnce(context.Background()); st.Sent != 1 { t.Fatalf("sent: %+v", st) } if hasDeadline { t.Fatal("the queue must not put a deadline on the whole send") } } func waitFor(t *testing.T, what string, cond func() bool) { t.Helper() deadline := time.Now().Add(3 * time.Second) for !cond() { if time.Now().After(deadline) { t.Fatalf("timed out waiting for %s", what) } time.Sleep(5 * time.Millisecond) } } func TestRunDrainsTheQueueInBatchesAndStopsOnCancel(t *testing.T) { e := setup(t) e.enqueue(t, 7) e.w.Batch = 2 e.w.Interval = time.Hour ctx, cancel := context.WithCancel(context.Background()) done := make(chan struct{}) go func() { e.w.Run(ctx); close(done) }() waitFor(t, "7 deliveries", func() bool { return e.snd.count() == 7 }) cancel() select { case <-done: case <-time.After(2 * time.Second): t.Fatal("Run did not stop") } } func TestWakeTriggersAnImmediateRun(t *testing.T) { e := setup(t) e.w.Interval = time.Hour ctx, cancel := context.WithCancel(context.Background()) defer cancel() go e.w.Run(ctx) time.Sleep(50 * time.Millisecond) // the first run finds nothing and goes idle e.enqueue(t, 1) e.w.Wake() waitFor(t, "woken delivery", func() bool { return e.snd.count() == 1 }) } func TestWakeBeforeRunAndRepeatedWakesDoNotBlock(t *testing.T) { e := setup(t) done := make(chan struct{}) go func() { for i := 0; i < 100; i++ { e.w.Wake() } close(done) }() select { case <-done: case <-time.After(time.Second): t.Fatal("Wake blocked") } } func TestIdleRunPollsForDueRetries(t *testing.T) { e := setup(t) e.enqueue(t, 1) fail := int32(1) e.snd.fn = func(context.Context, string, []string) error { if atomic.LoadInt32(&fail) == 1 { return errors.New("down") } return nil } e.w.Interval = 20 * time.Millisecond ctx, cancel := context.WithCancel(context.Background()) defer cancel() go e.w.Run(ctx) waitFor(t, "first failed attempt", func() bool { st, _ := e.s.OutboxStats(); return st.Retrying == 1 }) atomic.StoreInt32(&fail, 0) e.clk.set(t0.Add(2 * time.Minute)) // the backoff has passed; no Wake needed waitFor(t, "retry by polling", func() bool { return e.snd.count() == 1 }) } func TestSignHookSeesTheFinalMessageIncludingThePerRecipientHeaders(t *testing.T) { e := setup(t) e.enqueue(t, 1) var seen []byte e.w.Sign = func(msg []byte) ([]byte, error) { seen = append([]byte(nil), msg...) return append([]byte("X-Signed: yes\r\n"), msg...), nil } if st, _ := e.w.RunOnce(context.Background()); st.Sent != 1 { t.Fatalf("%+v", st) } if string(seen) != "X-Rcpt: a\r\nSubject: hi\r\n\r\nbody\r\n" { t.Fatalf("the signer must see prefix + message, so List-Unsubscribe is covered: %q", seen) } if string(e.snd.sent[0].Data) != "X-Signed: yes\r\nX-Rcpt: a\r\nSubject: hi\r\n\r\nbody\r\n" { t.Fatalf("the relay must get the signed message: %q", e.snd.sent[0].Data) } } func TestSigningFailureSendsTheMessageUnsigned(t *testing.T) { e := setup(t) e.enqueue(t, 1) e.w.Sign = func(msg []byte) ([]byte, error) { return nil, errors.New("hsm unavailable") } st, _ := e.w.RunOnce(context.Background()) if st.Sent != 1 || e.pending(t) != 0 { t.Fatalf("a signing error must not stop delivery: %+v", st) } if string(e.snd.sent[0].Data) != "X-Rcpt: a\r\nSubject: hi\r\n\r\nbody\r\n" { t.Fatalf("unsigned message expected: %q", e.snd.sent[0].Data) } } func TestEachAttemptIsSignedAgain(t *testing.T) { e := setup(t) e.enqueue(t, 1) calls := 0 e.w.Sign = func(msg []byte) ([]byte, error) { calls++; return msg, nil } failures := int32(1) e.snd.fn = func(context.Context, string, []string) error { if atomic.AddInt32(&failures, -1) >= 0 { return errors.New("down") } return nil } e.w.RunOnce(context.Background()) e.clk.set(t0.Add(2 * time.Minute)) e.w.RunOnce(context.Background()) if calls != 2 { t.Fatalf("a retry is signed afresh (new timestamp), signing calls = %d", calls) } } func TestARefusedSenderIsNotAMissingMailbox(t *testing.T) { // A relay that rejects the list's own sender address (5.1.8, 5.1.7, 5.1.0 or even 5.1.1 answering // MAIL FROM) says nothing about the recipients: the members must not be struck, and the mail must // stay visible as failed instead of vanishing. e := setup(t) l, _ := e.s.CreateList("dev", "example.org", "Dev") e.s.AddMember(l.ID, "ok@x.com", "") ms, _ := e.s.Members(l.ID) e.snd.fn = func(context.Context, string, []string) error { return &outbound.PhaseError{Phase: "MAIL", Err: &smtp.SMTPError{Code: 550, EnhancedCode: smtp.EnhancedCode{5, 1, 8}, Message: "Bad outbound sender"}} } for i := 0; i < 5; i++ { e.s.Enqueue([]byte("x"), []store.NewOutbound{{From: "dev+bounce-1-x@example.org", Rcpt: "ok@x.com", MemberID: ms[0].ID}}) st, _ := e.w.RunOnce(context.Background()) if st.Failed != 1 || st.Bounced != 0 || st.Dropped != 0 { t.Fatalf("round %d: %+v", i+1, st) } } if m, _ := e.s.MemberByID(ms[0].ID); m.BounceCount != 0 || m.Disabled { t.Fatalf("the member was struck for our own sender problem: %+v", m) } if stats, _ := e.s.OutboxStats(); stats.Failed != 5 { t.Fatalf("every delivery stays listed as failed: %+v", stats) } } func TestPurgeRunsOncePerIntervalNotOnEveryBatch(t *testing.T) { e := setup(t) var purges int32 e.w.purge = func() (int, error) { atomic.AddInt32(&purges, 1); return 0, nil } e.w.Batch = 1 e.enqueue(t, 3) for i := 0; i < 3; i++ { if st, err := e.w.RunOnce(context.Background()); err != nil || st.Sent != 1 { t.Fatalf("run %d: %+v %v", i, st, err) } } if n := atomic.LoadInt32(&purges); n != 1 { t.Fatalf("purge ran %d times in 3 batches, want 1", n) } e.clk.set(t0.Add(purgeEvery - time.Second)) e.w.RunOnce(context.Background()) if n := atomic.LoadInt32(&purges); n != 1 { t.Fatalf("purge ran again before the interval: %d", n) } e.clk.set(t0.Add(purgeEvery)) e.w.RunOnce(context.Background()) if n := atomic.LoadInt32(&purges); n != 2 { t.Fatalf("purge must run again after the interval: %d", n) } e.w.RunOnce(context.Background()) if n := atomic.LoadInt32(&purges); n != 2 { t.Fatalf("interval counts from the last purge: %d", n) } } func TestFailedPurgeIsRetriedOnTheNextRun(t *testing.T) { e := setup(t) var purges int32 e.w.purge = func() (int, error) { atomic.AddInt32(&purges, 1); return 0, errors.New("db busy") } e.w.RunOnce(context.Background()) e.w.RunOnce(context.Background()) if n := atomic.LoadInt32(&purges); n != 2 { t.Fatalf("a failed purge must not delay the next attempt: %d runs", n) } } func TestOneBodyReadServesEveryRecipientOfAMessage(t *testing.T) { e := setup(t) reads := map[int64]int{} var mu sync.Mutex e.w.readBody = func(id int64) ([]byte, error) { mu.Lock() reads[id]++ mu.Unlock() return e.s.OutboxData(id) } e.enqueue(t, 6) e.w.Concurrency = 6 if st, err := e.w.RunOnce(context.Background()); err != nil || st.Sent != 6 { t.Fatalf("run: %+v %v", st, err) } if len(reads) != 1 { t.Fatalf("reads = %v, want one message", reads) } for id, n := range reads { if n != 1 { t.Fatalf("message %d read %d times, want 1", id, n) } } for _, m := range e.snd.sent { if !strings.HasSuffix(string(m.Data), "Subject: hi\r\n\r\nbody\r\n") { t.Fatalf("data = %q", m.Data) } } // The cache lives for one run only: a later run reads again. e.enqueue(t, 1) e.w.RunOnce(context.Background()) total := 0 for _, n := range reads { total += n } if total != 2 { t.Fatalf("the next run must read its own body: %v", reads) } } func TestEachDistinctMessageIsReadOnce(t *testing.T) { e := setup(t) var calls int32 e.w.readBody = func(id int64) ([]byte, error) { atomic.AddInt32(&calls, 1); return e.s.OutboxData(id) } for i := 0; i < 3; i++ { e.enqueue(t, 2) } if st, _ := e.w.RunOnce(context.Background()); st.Sent != 6 { t.Fatalf("sent %d", st.Sent) } if n := atomic.LoadInt32(&calls); n != 3 { t.Fatalf("%d reads for 3 messages", n) } } func TestAMissingBodyIsDroppedForEveryRecipientWithOneRead(t *testing.T) { e := setup(t) var calls int32 e.w.readBody = func(int64) ([]byte, error) { atomic.AddInt32(&calls, 1); return nil, store.ErrNotFound } e.enqueue(t, 3) st, _ := e.w.RunOnce(context.Background()) if st.Dropped != 3 || e.snd.count() != 0 { t.Fatalf("stats %+v", st) } if n := atomic.LoadInt32(&calls); n != 1 { t.Fatalf("%d reads", n) } } func TestRateSpacesTheSendsOfAWorker(t *testing.T) { e := setup(t) e.enqueue(t, 5) e.w.Rate = 2 // one send every 500 ms f := &fakeTime{t: t0} e.w.pacerNow, e.w.pacerSleep = f.now, f.sleep st, _ := e.w.RunOnce(context.Background()) if st.Sent != 5 { t.Fatalf("%+v", st) } got := f.durations() // 5 deliveries: slots at 0, 1, 2, 3 and 4 intervals want := []time.Duration{500 * time.Millisecond, time.Second, 1500 * time.Millisecond, 2 * time.Second} if len(got) != len(want) { t.Fatalf("sleeps %v, want %v", got, want) } for i := range want { if got[i] != want[i] { t.Fatalf("sleeps %v, want %v", got, want) } } } func TestRateOfZeroNeverWaits(t *testing.T) { e := setup(t) e.enqueue(t, 5) f := &fakeTime{t: t0} e.w.pacerNow, e.w.pacerSleep = f.now, f.sleep e.w.RunOnce(context.Background()) if len(f.slept) != 0 || e.snd.count() != 5 { t.Fatalf("slept %v, sent %d", f.slept, e.snd.count()) } } func TestEverySendAttemptIsPacedEvenARetry(t *testing.T) { e := setup(t) e.enqueue(t, 1) e.w.Rate = 2 f := &fakeTime{t: t0} e.w.pacerNow, e.w.pacerSleep = f.now, f.sleep e.snd.fn = func(context.Context, string, []string) error { return errors.New("try later") } e.w.RunOnce(context.Background()) e.clk.set(t0.Add(time.Hour)) e.w.RunOnce(context.Background()) if e.snd.calls != 2 || len(f.slept) != 1 { t.Fatalf("calls=%d sleeps=%v: the second attempt shares the pacer with the first", e.snd.calls, f.slept) } } func TestPacingIsCancelledWithoutRecordingAnAttempt(t *testing.T) { e := setup(t) e.enqueue(t, 2) e.w.Rate = 1 e.w.Concurrency = 1 ctx, cancel := context.WithCancel(context.Background()) f := &fakeTime{t: t0} e.w.pacerNow = f.now e.w.pacerSleep = func(ctx context.Context, d time.Duration) error { cancel() // shutdown arrives while the delivery waits for its slot return ctx.Err() } st, _ := e.w.RunOnce(ctx) if st != (Stats{Sent: 1}) { t.Fatalf("stats %+v: the delivery that did not get its slot is not counted", st) } list, _ := e.s.ListOutbox(10) if len(list) != 1 || list[0].Attempts != 0 || list[0].LastError != "" { t.Fatalf("a shutdown while waiting is not a delivery failure: %+v", list) } if e.snd.calls != 1 { t.Fatalf("calls = %d: only the first delivery, which did not wait, was sent", e.snd.calls) } } func TestRateHoldsInRealTime(t *testing.T) { e := setup(t) e.enqueue(t, 20) e.w.Rate = 200 // 5 ms apart: 19 intervals take 95 ms start := time.Now() st, _ := e.w.RunOnce(context.Background()) took := time.Since(start) if st.Sent != 20 { t.Fatalf("%+v", st) } if took < 90*time.Millisecond || took > time.Second { t.Fatalf("20 sends at 200/s took %v, want about 95 ms", took) } }