parsimony/internal/queue/queue_test.go
tgiachi e6713962f9 feat(queue): background worker with retries, backoff and bounded concurrency
- Add Worker.RunOnce, Run and Wake over the durable outbox
- Send prefix plus shared message per recipient with a per-send timeout
- Retry temporary failures after 1m, 5m, 15m, 1h, 3h, then every 6h; give up after 3 days
- Drop permanent failures; count a hard bounce only for bad-mailbox replies
- Leave interrupted sends queued on shutdown
- Share the bounce threshold between router and worker
2026-11-09 15:47:14 +01:00

423 lines
12 KiB
Go

package queue
import (
"context"
"errors"
"strings"
"sync"
"sync/atomic"
"testing"
"time"
"github.com/emersion/go-smtp"
"codeberg.org/squidd/parsimony/internal/outbound"
"codeberg.org/squidd/parsimony/internal/store"
)
var t0 = time.Unix(1_900_000_000, 0)
// script is a Sender whose behaviour the test controls.
type script struct {
mu sync.Mutex
sent []outbound.Sent
fn func(ctx context.Context, from string, to []string) error // nil: succeed
calls int32
}
func (s *script) Send(ctx context.Context, from string, to []string, data []byte) error {
atomic.AddInt32(&s.calls, 1)
if s.fn != nil {
if err := s.fn(ctx, from, to); err != nil {
return err
}
}
s.mu.Lock()
defer s.mu.Unlock()
s.sent = append(s.sent, outbound.Sent{From: from, To: append([]string(nil), to...), Data: append([]byte(nil), data...)})
return nil
}
func (s *script) count() int {
s.mu.Lock()
defer s.mu.Unlock()
return len(s.sent)
}
// clock is a test clock that the worker goroutine may read while the test moves it.
type clock struct {
mu sync.Mutex
t time.Time
}
func (c *clock) now() time.Time {
c.mu.Lock()
defer c.mu.Unlock()
return c.t
}
func (c *clock) set(t time.Time) {
c.mu.Lock()
c.t = t
c.mu.Unlock()
}
type env struct {
s *store.Store
w *Worker
snd *script
clk *clock
}
func setup(t *testing.T) *env {
t.Helper()
s, err := store.Open(":memory:")
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { s.Close() })
clk := &clock{t: t0}
s.Now = clk.now
snd := &script{}
return &env{s: s, snd: snd, clk: clk, w: &Worker{Store: s, Sender: snd}}
}
func (e *env) enqueue(t *testing.T, n int) {
t.Helper()
var items []store.NewOutbound
for i := 0; i < n; i++ {
items = append(items, store.NewOutbound{From: "bounce@x.org", Rcpt: string(rune('a'+i)) + "@x.com", Prefix: []byte("X-Rcpt: " + string(rune('a'+i)) + "\r\n")})
}
if err := e.s.Enqueue([]byte("Subject: hi\r\n\r\nbody\r\n"), items); err != nil {
t.Fatal(err)
}
}
func (e *env) pending(t *testing.T) int {
t.Helper()
st, _ := e.s.OutboxStats()
return st.Pending
}
func TestDeliversPrefixPlusMessage(t *testing.T) {
e := setup(t)
e.enqueue(t, 2)
st, err := e.w.RunOnce(context.Background())
if err != nil || st.Sent != 2 {
t.Fatalf("run: %+v %v", st, err)
}
if e.pending(t) != 0 {
t.Fatal("sent items must leave the queue")
}
got := e.snd.sent[0]
if got.From != "bounce@x.org" || len(got.To) != 1 || got.To[0] != "a@x.com" {
t.Fatalf("envelope: %+v", got)
}
if string(got.Data) != "X-Rcpt: a\r\nSubject: hi\r\n\r\nbody\r\n" {
t.Fatalf("data = %q", got.Data)
}
if string(e.snd.sent[1].Data) != "X-Rcpt: b\r\nSubject: hi\r\n\r\nbody\r\n" {
t.Fatalf("each recipient has its own prefix: %q", e.snd.sent[1].Data)
}
}
func TestEmptyQueueIsFine(t *testing.T) {
e := setup(t)
if st, err := e.w.RunOnce(context.Background()); err != nil || st != (Stats{}) {
t.Fatalf("%+v %v", st, err)
}
}
func TestBackoffSchedule(t *testing.T) {
want := []time.Duration{time.Minute, 5 * time.Minute, 15 * time.Minute, time.Hour, 3 * time.Hour, 6 * time.Hour, 6 * time.Hour, 6 * time.Hour}
for i, w := range want {
if got := Backoff(i + 1); got != w {
t.Errorf("Backoff(%d) = %v, want %v", i+1, got, w)
}
}
if Backoff(0) != time.Minute || Backoff(-3) != time.Minute {
t.Error("nonsense input falls back to the first delay")
}
}
func TestTemporaryFailureIsRetriedLater(t *testing.T) {
e := setup(t)
e.enqueue(t, 1)
e.snd.fn = func(context.Context, string, []string) error { return errors.New("connection refused") }
st, _ := e.w.RunOnce(context.Background())
if st.Retried != 1 || st.Sent != 0 {
t.Fatalf("stats: %+v", st)
}
list, _ := e.s.ListOutbox(10)
if len(list) != 1 || list[0].Attempts != 1 || !strings.Contains(list[0].LastError, "connection refused") ||
!list[0].NextAt.Equal(t0.Add(time.Minute)) {
t.Fatalf("item: %+v", list)
}
// Not due yet: nothing is attempted.
before := atomic.LoadInt32(&e.snd.calls)
e.clk.set(t0.Add(59 * time.Second))
e.w.RunOnce(context.Background())
if atomic.LoadInt32(&e.snd.calls) != before {
t.Fatal("must wait for the backoff")
}
// Second failure waits longer.
e.clk.set(t0.Add(time.Minute))
e.w.RunOnce(context.Background())
list, _ = e.s.ListOutbox(10)
if list[0].Attempts != 2 || !list[0].NextAt.Equal(t0.Add(time.Minute+5*time.Minute)) {
t.Fatalf("second failure: %+v", list[0])
}
// The relay comes back.
e.snd.fn = nil
e.clk.set(t0.Add(10 * time.Minute))
st, _ = e.w.RunOnce(context.Background())
if st.Sent != 1 || e.pending(t) != 0 || e.snd.count() != 1 {
t.Fatalf("recovery: %+v pending=%d sent=%d", st, e.pending(t), e.snd.count())
}
}
func TestOneFailureDoesNotBlockTheOthers(t *testing.T) {
e := setup(t)
e.enqueue(t, 3)
e.snd.fn = func(_ context.Context, _ string, to []string) error {
if to[0] == "b@x.com" {
return errors.New("greylisted")
}
return nil
}
st, _ := e.w.RunOnce(context.Background())
if st.Sent != 2 || st.Retried != 1 || e.pending(t) != 1 {
t.Fatalf("%+v pending=%d", st, e.pending(t))
}
list, _ := e.s.ListOutbox(10)
if list[0].Rcpt != "b@x.com" {
t.Fatalf("the failed recipient stays queued, got %q", list[0].Rcpt)
}
}
func TestPermanentFailureDropsWithoutBouncing(t *testing.T) {
e := setup(t)
e.enqueue(t, 1)
e.snd.fn = func(context.Context, string, []string) error {
return &smtp.SMTPError{Code: 554, EnhancedCode: smtp.EnhancedCode{5, 7, 1}, Message: "relay access denied"}
}
st, _ := e.w.RunOnce(context.Background())
if st.Dropped != 1 || st.Bounced != 0 || e.pending(t) != 0 {
t.Fatalf("%+v pending=%d", st, e.pending(t))
}
}
func TestBadMailboxCountsAHardBounceAndDisablesAfterThree(t *testing.T) {
e := setup(t)
l, _ := e.s.CreateList("dev", "example.org", "Dev")
e.s.AddMember(l.ID, "gone@x.com", "")
ms, _ := e.s.Members(l.ID)
id := ms[0].ID
e.snd.fn = func(context.Context, string, []string) error {
return &smtp.SMTPError{Code: 550, EnhancedCode: smtp.EnhancedCode{5, 1, 1}, Message: "no such user"}
}
for i := 1; i <= 3; i++ {
e.s.Enqueue([]byte("Subject: x\r\n\r\nx"), []store.NewOutbound{{From: "b@x.org", Rcpt: "gone@x.com", MemberID: id}})
st, _ := e.w.RunOnce(context.Background())
if st.Dropped != 1 || st.Bounced != 1 {
t.Fatalf("round %d: %+v", i, st)
}
m, _ := e.s.MemberByID(id)
if m.BounceCount != i || m.Disabled != (i == 3) {
t.Fatalf("round %d: member %+v", i, m)
}
}
if e.pending(t) != 0 {
t.Fatal("permanent failures leave the queue")
}
}
func TestBadMailboxForSystemMailOrUnknownMemberIsHarmless(t *testing.T) {
e := setup(t)
e.snd.fn = func(context.Context, string, []string) error {
return &smtp.SMTPError{Code: 550, EnhancedCode: smtp.EnhancedCode{5, 1, 1}, Message: "no such user"}
}
e.s.Enqueue([]byte("x"), []store.NewOutbound{
{From: "b@x.org", Rcpt: "a@x.com", MemberID: 0}, // confirmation mail
{From: "b@x.org", Rcpt: "b@x.com", MemberID: 404}, // member was deleted meanwhile
})
st, err := e.w.RunOnce(context.Background())
if err != nil || st.Dropped != 2 || e.pending(t) != 0 {
t.Fatalf("%+v %v pending=%d", st, err, e.pending(t))
}
}
func TestGivesUpAfterMaxAge(t *testing.T) {
e := setup(t)
e.enqueue(t, 1)
e.snd.fn = func(context.Context, string, []string) error { return errors.New("still down") }
e.clk.set(t0.Add(MaxAge - time.Minute))
if st, _ := e.w.RunOnce(context.Background()); st.Retried != 1 {
t.Fatalf("just inside the limit is retried: %+v", st)
}
e.clk.set(t0.Add(MaxAge + time.Minute))
// The retry delay may not have passed yet; move far enough.
e.clk.set(e.clk.now().Add(7 * time.Hour))
st, _ := e.w.RunOnce(context.Background())
if st.Dropped != 1 || e.pending(t) != 0 {
t.Fatalf("too old: %+v pending=%d", st, e.pending(t))
}
}
func TestOldMailIsStillDeliveredIfTheRelayWorks(t *testing.T) {
e := setup(t)
e.enqueue(t, 1)
e.clk.set(t0.Add(MaxAge * 2))
if st, _ := e.w.RunOnce(context.Background()); st.Sent != 1 {
t.Fatalf("age only matters for failures: %+v", st)
}
}
func TestConcurrencyIsBounded(t *testing.T) {
e := setup(t)
e.enqueue(t, 10)
e.w.Concurrency = 3
var cur, max int32
e.snd.fn = func(ctx context.Context, _ string, _ []string) error {
n := atomic.AddInt32(&cur, 1)
for {
m := atomic.LoadInt32(&max)
if n <= m || atomic.CompareAndSwapInt32(&max, m, n) {
break
}
}
time.Sleep(20 * time.Millisecond)
atomic.AddInt32(&cur, -1)
return nil
}
st, _ := e.w.RunOnce(context.Background())
if st.Sent != 10 {
t.Fatalf("%+v", st)
}
if max != 3 {
t.Fatalf("max concurrent sends = %d, want 3", max)
}
}
func TestBatchLimitsOneRun(t *testing.T) {
e := setup(t)
e.enqueue(t, 5)
e.w.Batch = 2
st, _ := e.w.RunOnce(context.Background())
if st.Sent != 2 || e.pending(t) != 3 {
t.Fatalf("%+v pending=%d", st, e.pending(t))
}
}
func TestInterruptedSendStaysQueuedUntouched(t *testing.T) {
e := setup(t)
e.enqueue(t, 1)
ctx, cancel := context.WithCancel(context.Background())
e.snd.fn = func(ctx context.Context, _ string, _ []string) error {
cancel() // shutdown arrives while the send is in flight
<-ctx.Done()
return ctx.Err()
}
e.w.RunOnce(ctx)
list, _ := e.s.ListOutbox(10)
if len(list) != 1 || list[0].Attempts != 0 || list[0].LastError != "" {
t.Fatalf("a shutdown is not a delivery failure: %+v", list)
}
}
func TestSlowRelayTimesOutAndIsRetried(t *testing.T) {
e := setup(t)
e.enqueue(t, 1)
e.w.Timeout = 30 * time.Millisecond
e.snd.fn = func(ctx context.Context, _ string, _ []string) error {
<-ctx.Done()
return ctx.Err()
}
st, _ := e.w.RunOnce(context.Background())
if st.Retried != 1 {
t.Fatalf("a hung relay is a temporary failure: %+v", st)
}
list, _ := e.s.ListOutbox(10)
if !strings.Contains(list[0].LastError, "deadline") {
t.Fatalf("last error: %q", list[0].LastError)
}
}
func waitFor(t *testing.T, what string, cond func() bool) {
t.Helper()
deadline := time.Now().Add(3 * time.Second)
for !cond() {
if time.Now().After(deadline) {
t.Fatalf("timed out waiting for %s", what)
}
time.Sleep(5 * time.Millisecond)
}
}
func TestRunDrainsTheQueueInBatchesAndStopsOnCancel(t *testing.T) {
e := setup(t)
e.enqueue(t, 7)
e.w.Batch = 2
e.w.Interval = time.Hour
ctx, cancel := context.WithCancel(context.Background())
done := make(chan struct{})
go func() { e.w.Run(ctx); close(done) }()
waitFor(t, "7 deliveries", func() bool { return e.snd.count() == 7 })
cancel()
select {
case <-done:
case <-time.After(2 * time.Second):
t.Fatal("Run did not stop")
}
}
func TestWakeTriggersAnImmediateRun(t *testing.T) {
e := setup(t)
e.w.Interval = time.Hour
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
go e.w.Run(ctx)
time.Sleep(50 * time.Millisecond) // the first run finds nothing and goes idle
e.enqueue(t, 1)
e.w.Wake()
waitFor(t, "woken delivery", func() bool { return e.snd.count() == 1 })
}
func TestWakeBeforeRunAndRepeatedWakesDoNotBlock(t *testing.T) {
e := setup(t)
done := make(chan struct{})
go func() {
for i := 0; i < 100; i++ {
e.w.Wake()
}
close(done)
}()
select {
case <-done:
case <-time.After(time.Second):
t.Fatal("Wake blocked")
}
}
func TestIdleRunPollsForDueRetries(t *testing.T) {
e := setup(t)
e.enqueue(t, 1)
fail := int32(1)
e.snd.fn = func(context.Context, string, []string) error {
if atomic.LoadInt32(&fail) == 1 {
return errors.New("down")
}
return nil
}
e.w.Interval = 20 * time.Millisecond
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
go e.w.Run(ctx)
waitFor(t, "first failed attempt", func() bool { st, _ := e.s.OutboxStats(); return st.Retrying == 1 })
atomic.StoreInt32(&fail, 0)
e.clk.set(t0.Add(2 * time.Minute)) // the backoff has passed; no Wake needed
waitFor(t, "retry by polling", func() bool { return e.snd.count() == 1 })
}