diff --git a/internal/queue/queue.go b/internal/queue/queue.go new file mode 100644 index 0000000..acb8ae1 --- /dev/null +++ b/internal/queue/queue.go @@ -0,0 +1,211 @@ +// Package queue delivers the outbox to the SMTP relay in the background. +package queue + +import ( + "context" + "errors" + "log" + "sync" + "time" + + "codeberg.org/squidd/parsimony/internal/outbound" + "codeberg.org/squidd/parsimony/internal/store" +) + +// MaxAge is how long a failing delivery is retried before it is dropped. +const MaxAge = 72 * time.Hour + +// Defaults for the zero values of Worker fields. +const ( + defaultConcurrency = 4 + defaultBatch = 50 + defaultTimeout = time.Minute + defaultInterval = 5 * time.Second +) + +var backoff = []time.Duration{time.Minute, 5 * time.Minute, 15 * time.Minute, time.Hour, 3 * time.Hour, 6 * time.Hour} + +// Backoff returns how long to wait after the n-th consecutive failure (n >= 1). +func Backoff(failures int) time.Duration { + if failures < 1 { + failures = 1 + } + if failures > len(backoff) { + failures = len(backoff) + } + return backoff[failures-1] +} + +// Stats counts what one run did. +type Stats struct { + Sent int // delivered to the relay + Retried int // failed temporarily, scheduled again + Dropped int // removed: permanent failure or too old + Bounced int // dropped deliveries that counted as a hard bounce +} + +// Worker sends queued deliveries. Zero values of the optional fields mean defaults. +type Worker struct { + Store *store.Store + Sender outbound.Sender + Concurrency int // simultaneous sends + Batch int // deliveries taken per run + Timeout time.Duration // per send + Interval time.Duration // idle polling, to pick up retries that have become due + + wakeOnce sync.Once + wake chan struct{} + runMu sync.Mutex +} + +func (w *Worker) wakeCh() chan struct{} { + w.wakeOnce.Do(func() { w.wake = make(chan struct{}, 1) }) + return w.wake +} + +// Wake asks Run to look at the queue now. It never blocks. +func (w *Worker) Wake() { + select { + case w.wakeCh() <- struct{}{}: + default: + } +} + +// Run delivers until ctx is cancelled. +func (w *Worker) Run(ctx context.Context) { + interval := w.Interval + if interval <= 0 { + interval = defaultInterval + } + timer := time.NewTimer(0) + defer timer.Stop() + for { + select { + case <-ctx.Done(): + return + case <-w.wakeCh(): + case <-timer.C: + } + // Keep going while there is work, so a big post drains without waiting for the timer. + for ctx.Err() == nil { + st, err := w.RunOnce(ctx) + if err != nil { + log.Printf("queue: %v", err) + break + } + if st == (Stats{}) { + break + } + } + if !timer.Stop() { + select { + case <-timer.C: + default: + } + } + timer.Reset(interval) + } +} + +// RunOnce attempts every delivery that is due, up to Batch, with bounded concurrency. +func (w *Worker) RunOnce(ctx context.Context) (Stats, error) { + w.runMu.Lock() + defer w.runMu.Unlock() + + batch := w.Batch + if batch <= 0 { + batch = defaultBatch + } + conc := w.Concurrency + if conc <= 0 { + conc = defaultConcurrency + } + due, err := w.Store.DueOutbox(batch) + if err != nil { + return Stats{}, err + } + + var ( + mu sync.Mutex + stats Stats + wg sync.WaitGroup + sem = make(chan struct{}, conc) + ) + for _, it := range due { + if ctx.Err() != nil { + break + } + sem <- struct{}{} + wg.Add(1) + go func(it store.OutboxItem) { + defer wg.Done() + defer func() { <-sem }() + o := w.deliver(ctx, it) + mu.Lock() + stats.Sent += o.Sent + stats.Retried += o.Retried + stats.Dropped += o.Dropped + stats.Bounced += o.Bounced + mu.Unlock() + }(it) + } + wg.Wait() + return stats, nil +} + +func (w *Worker) deliver(ctx context.Context, it store.OutboxItem) Stats { + data, err := w.Store.OutboxData(it.MsgID) + if errors.Is(err, store.ErrNotFound) { + w.complete(it) + return Stats{Dropped: 1} + } + if err != nil { + log.Printf("queue: read message %d: %v", it.MsgID, err) + return Stats{} + } + msg := make([]byte, 0, len(it.Prefix)+len(data)) + msg = append(append(msg, it.Prefix...), data...) + + timeout := w.Timeout + if timeout <= 0 { + timeout = defaultTimeout + } + sctx, cancel := context.WithTimeout(ctx, timeout) + err = w.Sender.Send(sctx, it.From, []string{it.Rcpt}, msg) + cancel() + + switch { + case err == nil: + w.complete(it) + return Stats{Sent: 1} + case ctx.Err() != nil: + return Stats{} // shutting down: the delivery stays queued, unchanged + case outbound.IsPermanent(err): + out := Stats{Dropped: 1} + if outbound.IsBadMailbox(err) && it.MemberID != 0 { + if _, e := w.Store.RecordHardBounce(it.MemberID, store.BounceThreshold); e == nil { + out.Bounced = 1 + } + } + log.Printf("queue: drop %s: %v", it.Rcpt, err) + w.complete(it) + return out + } + now := w.Store.Now() + if now.Sub(it.CreatedAt) > MaxAge { + log.Printf("queue: give up on %s after %s: %v", it.Rcpt, MaxAge, err) + w.complete(it) + return Stats{Dropped: 1} + } + if e := w.Store.RetryOutbox(it.ID, now.Add(Backoff(it.Attempts+1)), err.Error()); e != nil { + log.Printf("queue: reschedule %d: %v", it.ID, e) + return Stats{} + } + return Stats{Retried: 1} +} + +func (w *Worker) complete(it store.OutboxItem) { + if err := w.Store.CompleteOutbox(it.ID); err != nil && !errors.Is(err, store.ErrNotFound) { + log.Printf("queue: remove %d: %v", it.ID, err) + } +} diff --git a/internal/queue/queue_test.go b/internal/queue/queue_test.go new file mode 100644 index 0000000..b8e185c --- /dev/null +++ b/internal/queue/queue_test.go @@ -0,0 +1,423 @@ +package queue + +import ( + "context" + "errors" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/emersion/go-smtp" + + "codeberg.org/squidd/parsimony/internal/outbound" + "codeberg.org/squidd/parsimony/internal/store" +) + +var t0 = time.Unix(1_900_000_000, 0) + +// script is a Sender whose behaviour the test controls. +type script struct { + mu sync.Mutex + sent []outbound.Sent + fn func(ctx context.Context, from string, to []string) error // nil: succeed + calls int32 +} + +func (s *script) Send(ctx context.Context, from string, to []string, data []byte) error { + atomic.AddInt32(&s.calls, 1) + if s.fn != nil { + if err := s.fn(ctx, from, to); err != nil { + return err + } + } + s.mu.Lock() + defer s.mu.Unlock() + s.sent = append(s.sent, outbound.Sent{From: from, To: append([]string(nil), to...), Data: append([]byte(nil), data...)}) + return nil +} + +func (s *script) count() int { + s.mu.Lock() + defer s.mu.Unlock() + return len(s.sent) +} + +// clock is a test clock that the worker goroutine may read while the test moves it. +type clock struct { + mu sync.Mutex + t time.Time +} + +func (c *clock) now() time.Time { + c.mu.Lock() + defer c.mu.Unlock() + return c.t +} + +func (c *clock) set(t time.Time) { + c.mu.Lock() + c.t = t + c.mu.Unlock() +} + +type env struct { + s *store.Store + w *Worker + snd *script + clk *clock +} + +func setup(t *testing.T) *env { + t.Helper() + s, err := store.Open(":memory:") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { s.Close() }) + clk := &clock{t: t0} + s.Now = clk.now + snd := &script{} + return &env{s: s, snd: snd, clk: clk, w: &Worker{Store: s, Sender: snd}} +} + +func (e *env) enqueue(t *testing.T, n int) { + t.Helper() + var items []store.NewOutbound + for i := 0; i < n; i++ { + items = append(items, store.NewOutbound{From: "bounce@x.org", Rcpt: string(rune('a'+i)) + "@x.com", Prefix: []byte("X-Rcpt: " + string(rune('a'+i)) + "\r\n")}) + } + if err := e.s.Enqueue([]byte("Subject: hi\r\n\r\nbody\r\n"), items); err != nil { + t.Fatal(err) + } +} + +func (e *env) pending(t *testing.T) int { + t.Helper() + st, _ := e.s.OutboxStats() + return st.Pending +} + +func TestDeliversPrefixPlusMessage(t *testing.T) { + e := setup(t) + e.enqueue(t, 2) + st, err := e.w.RunOnce(context.Background()) + if err != nil || st.Sent != 2 { + t.Fatalf("run: %+v %v", st, err) + } + if e.pending(t) != 0 { + t.Fatal("sent items must leave the queue") + } + got := e.snd.sent[0] + if got.From != "bounce@x.org" || len(got.To) != 1 || got.To[0] != "a@x.com" { + t.Fatalf("envelope: %+v", got) + } + if string(got.Data) != "X-Rcpt: a\r\nSubject: hi\r\n\r\nbody\r\n" { + t.Fatalf("data = %q", got.Data) + } + if string(e.snd.sent[1].Data) != "X-Rcpt: b\r\nSubject: hi\r\n\r\nbody\r\n" { + t.Fatalf("each recipient has its own prefix: %q", e.snd.sent[1].Data) + } +} + +func TestEmptyQueueIsFine(t *testing.T) { + e := setup(t) + if st, err := e.w.RunOnce(context.Background()); err != nil || st != (Stats{}) { + t.Fatalf("%+v %v", st, err) + } +} + +func TestBackoffSchedule(t *testing.T) { + want := []time.Duration{time.Minute, 5 * time.Minute, 15 * time.Minute, time.Hour, 3 * time.Hour, 6 * time.Hour, 6 * time.Hour, 6 * time.Hour} + for i, w := range want { + if got := Backoff(i + 1); got != w { + t.Errorf("Backoff(%d) = %v, want %v", i+1, got, w) + } + } + if Backoff(0) != time.Minute || Backoff(-3) != time.Minute { + t.Error("nonsense input falls back to the first delay") + } +} + +func TestTemporaryFailureIsRetriedLater(t *testing.T) { + e := setup(t) + e.enqueue(t, 1) + e.snd.fn = func(context.Context, string, []string) error { return errors.New("connection refused") } + st, _ := e.w.RunOnce(context.Background()) + if st.Retried != 1 || st.Sent != 0 { + t.Fatalf("stats: %+v", st) + } + list, _ := e.s.ListOutbox(10) + if len(list) != 1 || list[0].Attempts != 1 || !strings.Contains(list[0].LastError, "connection refused") || + !list[0].NextAt.Equal(t0.Add(time.Minute)) { + t.Fatalf("item: %+v", list) + } + + // Not due yet: nothing is attempted. + before := atomic.LoadInt32(&e.snd.calls) + e.clk.set(t0.Add(59 * time.Second)) + e.w.RunOnce(context.Background()) + if atomic.LoadInt32(&e.snd.calls) != before { + t.Fatal("must wait for the backoff") + } + + // Second failure waits longer. + e.clk.set(t0.Add(time.Minute)) + e.w.RunOnce(context.Background()) + list, _ = e.s.ListOutbox(10) + if list[0].Attempts != 2 || !list[0].NextAt.Equal(t0.Add(time.Minute+5*time.Minute)) { + t.Fatalf("second failure: %+v", list[0]) + } + + // The relay comes back. + e.snd.fn = nil + e.clk.set(t0.Add(10 * time.Minute)) + st, _ = e.w.RunOnce(context.Background()) + if st.Sent != 1 || e.pending(t) != 0 || e.snd.count() != 1 { + t.Fatalf("recovery: %+v pending=%d sent=%d", st, e.pending(t), e.snd.count()) + } +} + +func TestOneFailureDoesNotBlockTheOthers(t *testing.T) { + e := setup(t) + e.enqueue(t, 3) + e.snd.fn = func(_ context.Context, _ string, to []string) error { + if to[0] == "b@x.com" { + return errors.New("greylisted") + } + return nil + } + st, _ := e.w.RunOnce(context.Background()) + if st.Sent != 2 || st.Retried != 1 || e.pending(t) != 1 { + t.Fatalf("%+v pending=%d", st, e.pending(t)) + } + list, _ := e.s.ListOutbox(10) + if list[0].Rcpt != "b@x.com" { + t.Fatalf("the failed recipient stays queued, got %q", list[0].Rcpt) + } +} + +func TestPermanentFailureDropsWithoutBouncing(t *testing.T) { + e := setup(t) + e.enqueue(t, 1) + e.snd.fn = func(context.Context, string, []string) error { + return &smtp.SMTPError{Code: 554, EnhancedCode: smtp.EnhancedCode{5, 7, 1}, Message: "relay access denied"} + } + st, _ := e.w.RunOnce(context.Background()) + if st.Dropped != 1 || st.Bounced != 0 || e.pending(t) != 0 { + t.Fatalf("%+v pending=%d", st, e.pending(t)) + } +} + +func TestBadMailboxCountsAHardBounceAndDisablesAfterThree(t *testing.T) { + e := setup(t) + l, _ := e.s.CreateList("dev", "example.org", "Dev") + e.s.AddMember(l.ID, "gone@x.com", "") + ms, _ := e.s.Members(l.ID) + id := ms[0].ID + e.snd.fn = func(context.Context, string, []string) error { + return &smtp.SMTPError{Code: 550, EnhancedCode: smtp.EnhancedCode{5, 1, 1}, Message: "no such user"} + } + for i := 1; i <= 3; i++ { + e.s.Enqueue([]byte("Subject: x\r\n\r\nx"), []store.NewOutbound{{From: "b@x.org", Rcpt: "gone@x.com", MemberID: id}}) + st, _ := e.w.RunOnce(context.Background()) + if st.Dropped != 1 || st.Bounced != 1 { + t.Fatalf("round %d: %+v", i, st) + } + m, _ := e.s.MemberByID(id) + if m.BounceCount != i || m.Disabled != (i == 3) { + t.Fatalf("round %d: member %+v", i, m) + } + } + if e.pending(t) != 0 { + t.Fatal("permanent failures leave the queue") + } +} + +func TestBadMailboxForSystemMailOrUnknownMemberIsHarmless(t *testing.T) { + e := setup(t) + e.snd.fn = func(context.Context, string, []string) error { + return &smtp.SMTPError{Code: 550, EnhancedCode: smtp.EnhancedCode{5, 1, 1}, Message: "no such user"} + } + e.s.Enqueue([]byte("x"), []store.NewOutbound{ + {From: "b@x.org", Rcpt: "a@x.com", MemberID: 0}, // confirmation mail + {From: "b@x.org", Rcpt: "b@x.com", MemberID: 404}, // member was deleted meanwhile + }) + st, err := e.w.RunOnce(context.Background()) + if err != nil || st.Dropped != 2 || e.pending(t) != 0 { + t.Fatalf("%+v %v pending=%d", st, err, e.pending(t)) + } +} + +func TestGivesUpAfterMaxAge(t *testing.T) { + e := setup(t) + e.enqueue(t, 1) + e.snd.fn = func(context.Context, string, []string) error { return errors.New("still down") } + e.clk.set(t0.Add(MaxAge - time.Minute)) + if st, _ := e.w.RunOnce(context.Background()); st.Retried != 1 { + t.Fatalf("just inside the limit is retried: %+v", st) + } + e.clk.set(t0.Add(MaxAge + time.Minute)) + // The retry delay may not have passed yet; move far enough. + e.clk.set(e.clk.now().Add(7 * time.Hour)) + st, _ := e.w.RunOnce(context.Background()) + if st.Dropped != 1 || e.pending(t) != 0 { + t.Fatalf("too old: %+v pending=%d", st, e.pending(t)) + } +} + +func TestOldMailIsStillDeliveredIfTheRelayWorks(t *testing.T) { + e := setup(t) + e.enqueue(t, 1) + e.clk.set(t0.Add(MaxAge * 2)) + if st, _ := e.w.RunOnce(context.Background()); st.Sent != 1 { + t.Fatalf("age only matters for failures: %+v", st) + } +} + +func TestConcurrencyIsBounded(t *testing.T) { + e := setup(t) + e.enqueue(t, 10) + e.w.Concurrency = 3 + var cur, max int32 + e.snd.fn = func(ctx context.Context, _ string, _ []string) error { + n := atomic.AddInt32(&cur, 1) + for { + m := atomic.LoadInt32(&max) + if n <= m || atomic.CompareAndSwapInt32(&max, m, n) { + break + } + } + time.Sleep(20 * time.Millisecond) + atomic.AddInt32(&cur, -1) + return nil + } + st, _ := e.w.RunOnce(context.Background()) + if st.Sent != 10 { + t.Fatalf("%+v", st) + } + if max != 3 { + t.Fatalf("max concurrent sends = %d, want 3", max) + } +} + +func TestBatchLimitsOneRun(t *testing.T) { + e := setup(t) + e.enqueue(t, 5) + e.w.Batch = 2 + st, _ := e.w.RunOnce(context.Background()) + if st.Sent != 2 || e.pending(t) != 3 { + t.Fatalf("%+v pending=%d", st, e.pending(t)) + } +} + +func TestInterruptedSendStaysQueuedUntouched(t *testing.T) { + e := setup(t) + e.enqueue(t, 1) + ctx, cancel := context.WithCancel(context.Background()) + e.snd.fn = func(ctx context.Context, _ string, _ []string) error { + cancel() // shutdown arrives while the send is in flight + <-ctx.Done() + return ctx.Err() + } + e.w.RunOnce(ctx) + list, _ := e.s.ListOutbox(10) + if len(list) != 1 || list[0].Attempts != 0 || list[0].LastError != "" { + t.Fatalf("a shutdown is not a delivery failure: %+v", list) + } +} + +func TestSlowRelayTimesOutAndIsRetried(t *testing.T) { + e := setup(t) + e.enqueue(t, 1) + e.w.Timeout = 30 * time.Millisecond + e.snd.fn = func(ctx context.Context, _ string, _ []string) error { + <-ctx.Done() + return ctx.Err() + } + st, _ := e.w.RunOnce(context.Background()) + if st.Retried != 1 { + t.Fatalf("a hung relay is a temporary failure: %+v", st) + } + list, _ := e.s.ListOutbox(10) + if !strings.Contains(list[0].LastError, "deadline") { + t.Fatalf("last error: %q", list[0].LastError) + } +} + +func waitFor(t *testing.T, what string, cond func() bool) { + t.Helper() + deadline := time.Now().Add(3 * time.Second) + for !cond() { + if time.Now().After(deadline) { + t.Fatalf("timed out waiting for %s", what) + } + time.Sleep(5 * time.Millisecond) + } +} + +func TestRunDrainsTheQueueInBatchesAndStopsOnCancel(t *testing.T) { + e := setup(t) + e.enqueue(t, 7) + e.w.Batch = 2 + e.w.Interval = time.Hour + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + go func() { e.w.Run(ctx); close(done) }() + waitFor(t, "7 deliveries", func() bool { return e.snd.count() == 7 }) + cancel() + select { + case <-done: + case <-time.After(2 * time.Second): + t.Fatal("Run did not stop") + } +} + +func TestWakeTriggersAnImmediateRun(t *testing.T) { + e := setup(t) + e.w.Interval = time.Hour + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + go e.w.Run(ctx) + time.Sleep(50 * time.Millisecond) // the first run finds nothing and goes idle + e.enqueue(t, 1) + e.w.Wake() + waitFor(t, "woken delivery", func() bool { return e.snd.count() == 1 }) +} + +func TestWakeBeforeRunAndRepeatedWakesDoNotBlock(t *testing.T) { + e := setup(t) + done := make(chan struct{}) + go func() { + for i := 0; i < 100; i++ { + e.w.Wake() + } + close(done) + }() + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("Wake blocked") + } +} + +func TestIdleRunPollsForDueRetries(t *testing.T) { + e := setup(t) + e.enqueue(t, 1) + fail := int32(1) + e.snd.fn = func(context.Context, string, []string) error { + if atomic.LoadInt32(&fail) == 1 { + return errors.New("down") + } + return nil + } + e.w.Interval = 20 * time.Millisecond + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + go e.w.Run(ctx) + waitFor(t, "first failed attempt", func() bool { st, _ := e.s.OutboxStats(); return st.Retrying == 1 }) + atomic.StoreInt32(&fail, 0) + e.clk.set(t0.Add(2 * time.Minute)) // the backoff has passed; no Wake needed + waitFor(t, "retry by polling", func() bool { return e.snd.count() == 1 }) +} diff --git a/internal/router/commands.go b/internal/router/commands.go index 5e3b131..7b2e0f7 100644 --- a/internal/router/commands.go +++ b/internal/router/commands.go @@ -18,9 +18,6 @@ import ( "codeberg.org/squidd/parsimony/internal/store" ) -// bounceThreshold is the number of hard bounces after which a member is disabled. -const bounceThreshold = 3 - var statusRe = regexp.MustCompile(`(?mi)^Status:\s*([245])\.\d+\.\d+`) func (r *Router) secret() ([]byte, error) { @@ -170,7 +167,7 @@ func (r *Router) bounce(l store.List, tag string, data []byte) error { return err } if sm := statusRe.FindSubmatch(data); sm != nil && string(sm[1]) == "5" { - _, err = r.store.RecordHardBounce(m.ID, bounceThreshold) + _, err = r.store.RecordHardBounce(m.ID, store.BounceThreshold) } return err } diff --git a/internal/store/store.go b/internal/store/store.go index e7fee0f..ec595f1 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -41,6 +41,9 @@ const ( ActionLeave = "leave" ) +// BounceThreshold is the number of hard bounces after which a member is disabled. +const BounceThreshold = 3 + const ( pendingTTL = 72 * time.Hour pendingRateLimit = time.Hour