feat(queue): background worker with retries, backoff and bounded concurrency
- Add Worker.RunOnce, Run and Wake over the durable outbox - Send prefix plus shared message per recipient with a per-send timeout - Retry temporary failures after 1m, 5m, 15m, 1h, 3h, then every 6h; give up after 3 days - Drop permanent failures; count a hard bounce only for bad-mailbox replies - Leave interrupted sends queued on shutdown - Share the bounce threshold between router and worker
This commit is contained in:
parent
ecbab4d18c
commit
e6713962f9
4 changed files with 638 additions and 4 deletions
211
internal/queue/queue.go
Normal file
211
internal/queue/queue.go
Normal file
|
|
@ -0,0 +1,211 @@
|
|||
// Package queue delivers the outbox to the SMTP relay in the background.
|
||||
package queue
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"log"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"codeberg.org/squidd/parsimony/internal/outbound"
|
||||
"codeberg.org/squidd/parsimony/internal/store"
|
||||
)
|
||||
|
||||
// MaxAge is how long a failing delivery is retried before it is dropped.
|
||||
const MaxAge = 72 * time.Hour
|
||||
|
||||
// Defaults for the zero values of Worker fields.
|
||||
const (
|
||||
defaultConcurrency = 4
|
||||
defaultBatch = 50
|
||||
defaultTimeout = time.Minute
|
||||
defaultInterval = 5 * time.Second
|
||||
)
|
||||
|
||||
var backoff = []time.Duration{time.Minute, 5 * time.Minute, 15 * time.Minute, time.Hour, 3 * time.Hour, 6 * time.Hour}
|
||||
|
||||
// Backoff returns how long to wait after the n-th consecutive failure (n >= 1).
|
||||
func Backoff(failures int) time.Duration {
|
||||
if failures < 1 {
|
||||
failures = 1
|
||||
}
|
||||
if failures > len(backoff) {
|
||||
failures = len(backoff)
|
||||
}
|
||||
return backoff[failures-1]
|
||||
}
|
||||
|
||||
// Stats counts what one run did.
|
||||
type Stats struct {
|
||||
Sent int // delivered to the relay
|
||||
Retried int // failed temporarily, scheduled again
|
||||
Dropped int // removed: permanent failure or too old
|
||||
Bounced int // dropped deliveries that counted as a hard bounce
|
||||
}
|
||||
|
||||
// Worker sends queued deliveries. Zero values of the optional fields mean defaults.
|
||||
type Worker struct {
|
||||
Store *store.Store
|
||||
Sender outbound.Sender
|
||||
Concurrency int // simultaneous sends
|
||||
Batch int // deliveries taken per run
|
||||
Timeout time.Duration // per send
|
||||
Interval time.Duration // idle polling, to pick up retries that have become due
|
||||
|
||||
wakeOnce sync.Once
|
||||
wake chan struct{}
|
||||
runMu sync.Mutex
|
||||
}
|
||||
|
||||
func (w *Worker) wakeCh() chan struct{} {
|
||||
w.wakeOnce.Do(func() { w.wake = make(chan struct{}, 1) })
|
||||
return w.wake
|
||||
}
|
||||
|
||||
// Wake asks Run to look at the queue now. It never blocks.
|
||||
func (w *Worker) Wake() {
|
||||
select {
|
||||
case w.wakeCh() <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
// Run delivers until ctx is cancelled.
|
||||
func (w *Worker) Run(ctx context.Context) {
|
||||
interval := w.Interval
|
||||
if interval <= 0 {
|
||||
interval = defaultInterval
|
||||
}
|
||||
timer := time.NewTimer(0)
|
||||
defer timer.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-w.wakeCh():
|
||||
case <-timer.C:
|
||||
}
|
||||
// Keep going while there is work, so a big post drains without waiting for the timer.
|
||||
for ctx.Err() == nil {
|
||||
st, err := w.RunOnce(ctx)
|
||||
if err != nil {
|
||||
log.Printf("queue: %v", err)
|
||||
break
|
||||
}
|
||||
if st == (Stats{}) {
|
||||
break
|
||||
}
|
||||
}
|
||||
if !timer.Stop() {
|
||||
select {
|
||||
case <-timer.C:
|
||||
default:
|
||||
}
|
||||
}
|
||||
timer.Reset(interval)
|
||||
}
|
||||
}
|
||||
|
||||
// RunOnce attempts every delivery that is due, up to Batch, with bounded concurrency.
|
||||
func (w *Worker) RunOnce(ctx context.Context) (Stats, error) {
|
||||
w.runMu.Lock()
|
||||
defer w.runMu.Unlock()
|
||||
|
||||
batch := w.Batch
|
||||
if batch <= 0 {
|
||||
batch = defaultBatch
|
||||
}
|
||||
conc := w.Concurrency
|
||||
if conc <= 0 {
|
||||
conc = defaultConcurrency
|
||||
}
|
||||
due, err := w.Store.DueOutbox(batch)
|
||||
if err != nil {
|
||||
return Stats{}, err
|
||||
}
|
||||
|
||||
var (
|
||||
mu sync.Mutex
|
||||
stats Stats
|
||||
wg sync.WaitGroup
|
||||
sem = make(chan struct{}, conc)
|
||||
)
|
||||
for _, it := range due {
|
||||
if ctx.Err() != nil {
|
||||
break
|
||||
}
|
||||
sem <- struct{}{}
|
||||
wg.Add(1)
|
||||
go func(it store.OutboxItem) {
|
||||
defer wg.Done()
|
||||
defer func() { <-sem }()
|
||||
o := w.deliver(ctx, it)
|
||||
mu.Lock()
|
||||
stats.Sent += o.Sent
|
||||
stats.Retried += o.Retried
|
||||
stats.Dropped += o.Dropped
|
||||
stats.Bounced += o.Bounced
|
||||
mu.Unlock()
|
||||
}(it)
|
||||
}
|
||||
wg.Wait()
|
||||
return stats, nil
|
||||
}
|
||||
|
||||
func (w *Worker) deliver(ctx context.Context, it store.OutboxItem) Stats {
|
||||
data, err := w.Store.OutboxData(it.MsgID)
|
||||
if errors.Is(err, store.ErrNotFound) {
|
||||
w.complete(it)
|
||||
return Stats{Dropped: 1}
|
||||
}
|
||||
if err != nil {
|
||||
log.Printf("queue: read message %d: %v", it.MsgID, err)
|
||||
return Stats{}
|
||||
}
|
||||
msg := make([]byte, 0, len(it.Prefix)+len(data))
|
||||
msg = append(append(msg, it.Prefix...), data...)
|
||||
|
||||
timeout := w.Timeout
|
||||
if timeout <= 0 {
|
||||
timeout = defaultTimeout
|
||||
}
|
||||
sctx, cancel := context.WithTimeout(ctx, timeout)
|
||||
err = w.Sender.Send(sctx, it.From, []string{it.Rcpt}, msg)
|
||||
cancel()
|
||||
|
||||
switch {
|
||||
case err == nil:
|
||||
w.complete(it)
|
||||
return Stats{Sent: 1}
|
||||
case ctx.Err() != nil:
|
||||
return Stats{} // shutting down: the delivery stays queued, unchanged
|
||||
case outbound.IsPermanent(err):
|
||||
out := Stats{Dropped: 1}
|
||||
if outbound.IsBadMailbox(err) && it.MemberID != 0 {
|
||||
if _, e := w.Store.RecordHardBounce(it.MemberID, store.BounceThreshold); e == nil {
|
||||
out.Bounced = 1
|
||||
}
|
||||
}
|
||||
log.Printf("queue: drop %s: %v", it.Rcpt, err)
|
||||
w.complete(it)
|
||||
return out
|
||||
}
|
||||
now := w.Store.Now()
|
||||
if now.Sub(it.CreatedAt) > MaxAge {
|
||||
log.Printf("queue: give up on %s after %s: %v", it.Rcpt, MaxAge, err)
|
||||
w.complete(it)
|
||||
return Stats{Dropped: 1}
|
||||
}
|
||||
if e := w.Store.RetryOutbox(it.ID, now.Add(Backoff(it.Attempts+1)), err.Error()); e != nil {
|
||||
log.Printf("queue: reschedule %d: %v", it.ID, e)
|
||||
return Stats{}
|
||||
}
|
||||
return Stats{Retried: 1}
|
||||
}
|
||||
|
||||
func (w *Worker) complete(it store.OutboxItem) {
|
||||
if err := w.Store.CompleteOutbox(it.ID); err != nil && !errors.Is(err, store.ErrNotFound) {
|
||||
log.Printf("queue: remove %d: %v", it.ID, err)
|
||||
}
|
||||
}
|
||||
423
internal/queue/queue_test.go
Normal file
423
internal/queue/queue_test.go
Normal file
|
|
@ -0,0 +1,423 @@
|
|||
package queue
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/emersion/go-smtp"
|
||||
|
||||
"codeberg.org/squidd/parsimony/internal/outbound"
|
||||
"codeberg.org/squidd/parsimony/internal/store"
|
||||
)
|
||||
|
||||
var t0 = time.Unix(1_900_000_000, 0)
|
||||
|
||||
// script is a Sender whose behaviour the test controls.
|
||||
type script struct {
|
||||
mu sync.Mutex
|
||||
sent []outbound.Sent
|
||||
fn func(ctx context.Context, from string, to []string) error // nil: succeed
|
||||
calls int32
|
||||
}
|
||||
|
||||
func (s *script) Send(ctx context.Context, from string, to []string, data []byte) error {
|
||||
atomic.AddInt32(&s.calls, 1)
|
||||
if s.fn != nil {
|
||||
if err := s.fn(ctx, from, to); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.sent = append(s.sent, outbound.Sent{From: from, To: append([]string(nil), to...), Data: append([]byte(nil), data...)})
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *script) count() int {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return len(s.sent)
|
||||
}
|
||||
|
||||
// clock is a test clock that the worker goroutine may read while the test moves it.
|
||||
type clock struct {
|
||||
mu sync.Mutex
|
||||
t time.Time
|
||||
}
|
||||
|
||||
func (c *clock) now() time.Time {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
return c.t
|
||||
}
|
||||
|
||||
func (c *clock) set(t time.Time) {
|
||||
c.mu.Lock()
|
||||
c.t = t
|
||||
c.mu.Unlock()
|
||||
}
|
||||
|
||||
type env struct {
|
||||
s *store.Store
|
||||
w *Worker
|
||||
snd *script
|
||||
clk *clock
|
||||
}
|
||||
|
||||
func setup(t *testing.T) *env {
|
||||
t.Helper()
|
||||
s, err := store.Open(":memory:")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { s.Close() })
|
||||
clk := &clock{t: t0}
|
||||
s.Now = clk.now
|
||||
snd := &script{}
|
||||
return &env{s: s, snd: snd, clk: clk, w: &Worker{Store: s, Sender: snd}}
|
||||
}
|
||||
|
||||
func (e *env) enqueue(t *testing.T, n int) {
|
||||
t.Helper()
|
||||
var items []store.NewOutbound
|
||||
for i := 0; i < n; i++ {
|
||||
items = append(items, store.NewOutbound{From: "bounce@x.org", Rcpt: string(rune('a'+i)) + "@x.com", Prefix: []byte("X-Rcpt: " + string(rune('a'+i)) + "\r\n")})
|
||||
}
|
||||
if err := e.s.Enqueue([]byte("Subject: hi\r\n\r\nbody\r\n"), items); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func (e *env) pending(t *testing.T) int {
|
||||
t.Helper()
|
||||
st, _ := e.s.OutboxStats()
|
||||
return st.Pending
|
||||
}
|
||||
|
||||
func TestDeliversPrefixPlusMessage(t *testing.T) {
|
||||
e := setup(t)
|
||||
e.enqueue(t, 2)
|
||||
st, err := e.w.RunOnce(context.Background())
|
||||
if err != nil || st.Sent != 2 {
|
||||
t.Fatalf("run: %+v %v", st, err)
|
||||
}
|
||||
if e.pending(t) != 0 {
|
||||
t.Fatal("sent items must leave the queue")
|
||||
}
|
||||
got := e.snd.sent[0]
|
||||
if got.From != "bounce@x.org" || len(got.To) != 1 || got.To[0] != "a@x.com" {
|
||||
t.Fatalf("envelope: %+v", got)
|
||||
}
|
||||
if string(got.Data) != "X-Rcpt: a\r\nSubject: hi\r\n\r\nbody\r\n" {
|
||||
t.Fatalf("data = %q", got.Data)
|
||||
}
|
||||
if string(e.snd.sent[1].Data) != "X-Rcpt: b\r\nSubject: hi\r\n\r\nbody\r\n" {
|
||||
t.Fatalf("each recipient has its own prefix: %q", e.snd.sent[1].Data)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEmptyQueueIsFine(t *testing.T) {
|
||||
e := setup(t)
|
||||
if st, err := e.w.RunOnce(context.Background()); err != nil || st != (Stats{}) {
|
||||
t.Fatalf("%+v %v", st, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBackoffSchedule(t *testing.T) {
|
||||
want := []time.Duration{time.Minute, 5 * time.Minute, 15 * time.Minute, time.Hour, 3 * time.Hour, 6 * time.Hour, 6 * time.Hour, 6 * time.Hour}
|
||||
for i, w := range want {
|
||||
if got := Backoff(i + 1); got != w {
|
||||
t.Errorf("Backoff(%d) = %v, want %v", i+1, got, w)
|
||||
}
|
||||
}
|
||||
if Backoff(0) != time.Minute || Backoff(-3) != time.Minute {
|
||||
t.Error("nonsense input falls back to the first delay")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTemporaryFailureIsRetriedLater(t *testing.T) {
|
||||
e := setup(t)
|
||||
e.enqueue(t, 1)
|
||||
e.snd.fn = func(context.Context, string, []string) error { return errors.New("connection refused") }
|
||||
st, _ := e.w.RunOnce(context.Background())
|
||||
if st.Retried != 1 || st.Sent != 0 {
|
||||
t.Fatalf("stats: %+v", st)
|
||||
}
|
||||
list, _ := e.s.ListOutbox(10)
|
||||
if len(list) != 1 || list[0].Attempts != 1 || !strings.Contains(list[0].LastError, "connection refused") ||
|
||||
!list[0].NextAt.Equal(t0.Add(time.Minute)) {
|
||||
t.Fatalf("item: %+v", list)
|
||||
}
|
||||
|
||||
// Not due yet: nothing is attempted.
|
||||
before := atomic.LoadInt32(&e.snd.calls)
|
||||
e.clk.set(t0.Add(59 * time.Second))
|
||||
e.w.RunOnce(context.Background())
|
||||
if atomic.LoadInt32(&e.snd.calls) != before {
|
||||
t.Fatal("must wait for the backoff")
|
||||
}
|
||||
|
||||
// Second failure waits longer.
|
||||
e.clk.set(t0.Add(time.Minute))
|
||||
e.w.RunOnce(context.Background())
|
||||
list, _ = e.s.ListOutbox(10)
|
||||
if list[0].Attempts != 2 || !list[0].NextAt.Equal(t0.Add(time.Minute+5*time.Minute)) {
|
||||
t.Fatalf("second failure: %+v", list[0])
|
||||
}
|
||||
|
||||
// The relay comes back.
|
||||
e.snd.fn = nil
|
||||
e.clk.set(t0.Add(10 * time.Minute))
|
||||
st, _ = e.w.RunOnce(context.Background())
|
||||
if st.Sent != 1 || e.pending(t) != 0 || e.snd.count() != 1 {
|
||||
t.Fatalf("recovery: %+v pending=%d sent=%d", st, e.pending(t), e.snd.count())
|
||||
}
|
||||
}
|
||||
|
||||
func TestOneFailureDoesNotBlockTheOthers(t *testing.T) {
|
||||
e := setup(t)
|
||||
e.enqueue(t, 3)
|
||||
e.snd.fn = func(_ context.Context, _ string, to []string) error {
|
||||
if to[0] == "b@x.com" {
|
||||
return errors.New("greylisted")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
st, _ := e.w.RunOnce(context.Background())
|
||||
if st.Sent != 2 || st.Retried != 1 || e.pending(t) != 1 {
|
||||
t.Fatalf("%+v pending=%d", st, e.pending(t))
|
||||
}
|
||||
list, _ := e.s.ListOutbox(10)
|
||||
if list[0].Rcpt != "b@x.com" {
|
||||
t.Fatalf("the failed recipient stays queued, got %q", list[0].Rcpt)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPermanentFailureDropsWithoutBouncing(t *testing.T) {
|
||||
e := setup(t)
|
||||
e.enqueue(t, 1)
|
||||
e.snd.fn = func(context.Context, string, []string) error {
|
||||
return &smtp.SMTPError{Code: 554, EnhancedCode: smtp.EnhancedCode{5, 7, 1}, Message: "relay access denied"}
|
||||
}
|
||||
st, _ := e.w.RunOnce(context.Background())
|
||||
if st.Dropped != 1 || st.Bounced != 0 || e.pending(t) != 0 {
|
||||
t.Fatalf("%+v pending=%d", st, e.pending(t))
|
||||
}
|
||||
}
|
||||
|
||||
func TestBadMailboxCountsAHardBounceAndDisablesAfterThree(t *testing.T) {
|
||||
e := setup(t)
|
||||
l, _ := e.s.CreateList("dev", "example.org", "Dev")
|
||||
e.s.AddMember(l.ID, "gone@x.com", "")
|
||||
ms, _ := e.s.Members(l.ID)
|
||||
id := ms[0].ID
|
||||
e.snd.fn = func(context.Context, string, []string) error {
|
||||
return &smtp.SMTPError{Code: 550, EnhancedCode: smtp.EnhancedCode{5, 1, 1}, Message: "no such user"}
|
||||
}
|
||||
for i := 1; i <= 3; i++ {
|
||||
e.s.Enqueue([]byte("Subject: x\r\n\r\nx"), []store.NewOutbound{{From: "b@x.org", Rcpt: "gone@x.com", MemberID: id}})
|
||||
st, _ := e.w.RunOnce(context.Background())
|
||||
if st.Dropped != 1 || st.Bounced != 1 {
|
||||
t.Fatalf("round %d: %+v", i, st)
|
||||
}
|
||||
m, _ := e.s.MemberByID(id)
|
||||
if m.BounceCount != i || m.Disabled != (i == 3) {
|
||||
t.Fatalf("round %d: member %+v", i, m)
|
||||
}
|
||||
}
|
||||
if e.pending(t) != 0 {
|
||||
t.Fatal("permanent failures leave the queue")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBadMailboxForSystemMailOrUnknownMemberIsHarmless(t *testing.T) {
|
||||
e := setup(t)
|
||||
e.snd.fn = func(context.Context, string, []string) error {
|
||||
return &smtp.SMTPError{Code: 550, EnhancedCode: smtp.EnhancedCode{5, 1, 1}, Message: "no such user"}
|
||||
}
|
||||
e.s.Enqueue([]byte("x"), []store.NewOutbound{
|
||||
{From: "b@x.org", Rcpt: "a@x.com", MemberID: 0}, // confirmation mail
|
||||
{From: "b@x.org", Rcpt: "b@x.com", MemberID: 404}, // member was deleted meanwhile
|
||||
})
|
||||
st, err := e.w.RunOnce(context.Background())
|
||||
if err != nil || st.Dropped != 2 || e.pending(t) != 0 {
|
||||
t.Fatalf("%+v %v pending=%d", st, err, e.pending(t))
|
||||
}
|
||||
}
|
||||
|
||||
func TestGivesUpAfterMaxAge(t *testing.T) {
|
||||
e := setup(t)
|
||||
e.enqueue(t, 1)
|
||||
e.snd.fn = func(context.Context, string, []string) error { return errors.New("still down") }
|
||||
e.clk.set(t0.Add(MaxAge - time.Minute))
|
||||
if st, _ := e.w.RunOnce(context.Background()); st.Retried != 1 {
|
||||
t.Fatalf("just inside the limit is retried: %+v", st)
|
||||
}
|
||||
e.clk.set(t0.Add(MaxAge + time.Minute))
|
||||
// The retry delay may not have passed yet; move far enough.
|
||||
e.clk.set(e.clk.now().Add(7 * time.Hour))
|
||||
st, _ := e.w.RunOnce(context.Background())
|
||||
if st.Dropped != 1 || e.pending(t) != 0 {
|
||||
t.Fatalf("too old: %+v pending=%d", st, e.pending(t))
|
||||
}
|
||||
}
|
||||
|
||||
func TestOldMailIsStillDeliveredIfTheRelayWorks(t *testing.T) {
|
||||
e := setup(t)
|
||||
e.enqueue(t, 1)
|
||||
e.clk.set(t0.Add(MaxAge * 2))
|
||||
if st, _ := e.w.RunOnce(context.Background()); st.Sent != 1 {
|
||||
t.Fatalf("age only matters for failures: %+v", st)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConcurrencyIsBounded(t *testing.T) {
|
||||
e := setup(t)
|
||||
e.enqueue(t, 10)
|
||||
e.w.Concurrency = 3
|
||||
var cur, max int32
|
||||
e.snd.fn = func(ctx context.Context, _ string, _ []string) error {
|
||||
n := atomic.AddInt32(&cur, 1)
|
||||
for {
|
||||
m := atomic.LoadInt32(&max)
|
||||
if n <= m || atomic.CompareAndSwapInt32(&max, m, n) {
|
||||
break
|
||||
}
|
||||
}
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
atomic.AddInt32(&cur, -1)
|
||||
return nil
|
||||
}
|
||||
st, _ := e.w.RunOnce(context.Background())
|
||||
if st.Sent != 10 {
|
||||
t.Fatalf("%+v", st)
|
||||
}
|
||||
if max != 3 {
|
||||
t.Fatalf("max concurrent sends = %d, want 3", max)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBatchLimitsOneRun(t *testing.T) {
|
||||
e := setup(t)
|
||||
e.enqueue(t, 5)
|
||||
e.w.Batch = 2
|
||||
st, _ := e.w.RunOnce(context.Background())
|
||||
if st.Sent != 2 || e.pending(t) != 3 {
|
||||
t.Fatalf("%+v pending=%d", st, e.pending(t))
|
||||
}
|
||||
}
|
||||
|
||||
func TestInterruptedSendStaysQueuedUntouched(t *testing.T) {
|
||||
e := setup(t)
|
||||
e.enqueue(t, 1)
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
e.snd.fn = func(ctx context.Context, _ string, _ []string) error {
|
||||
cancel() // shutdown arrives while the send is in flight
|
||||
<-ctx.Done()
|
||||
return ctx.Err()
|
||||
}
|
||||
e.w.RunOnce(ctx)
|
||||
list, _ := e.s.ListOutbox(10)
|
||||
if len(list) != 1 || list[0].Attempts != 0 || list[0].LastError != "" {
|
||||
t.Fatalf("a shutdown is not a delivery failure: %+v", list)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSlowRelayTimesOutAndIsRetried(t *testing.T) {
|
||||
e := setup(t)
|
||||
e.enqueue(t, 1)
|
||||
e.w.Timeout = 30 * time.Millisecond
|
||||
e.snd.fn = func(ctx context.Context, _ string, _ []string) error {
|
||||
<-ctx.Done()
|
||||
return ctx.Err()
|
||||
}
|
||||
st, _ := e.w.RunOnce(context.Background())
|
||||
if st.Retried != 1 {
|
||||
t.Fatalf("a hung relay is a temporary failure: %+v", st)
|
||||
}
|
||||
list, _ := e.s.ListOutbox(10)
|
||||
if !strings.Contains(list[0].LastError, "deadline") {
|
||||
t.Fatalf("last error: %q", list[0].LastError)
|
||||
}
|
||||
}
|
||||
|
||||
func waitFor(t *testing.T, what string, cond func() bool) {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(3 * time.Second)
|
||||
for !cond() {
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatalf("timed out waiting for %s", what)
|
||||
}
|
||||
time.Sleep(5 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunDrainsTheQueueInBatchesAndStopsOnCancel(t *testing.T) {
|
||||
e := setup(t)
|
||||
e.enqueue(t, 7)
|
||||
e.w.Batch = 2
|
||||
e.w.Interval = time.Hour
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
done := make(chan struct{})
|
||||
go func() { e.w.Run(ctx); close(done) }()
|
||||
waitFor(t, "7 deliveries", func() bool { return e.snd.count() == 7 })
|
||||
cancel()
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("Run did not stop")
|
||||
}
|
||||
}
|
||||
|
||||
func TestWakeTriggersAnImmediateRun(t *testing.T) {
|
||||
e := setup(t)
|
||||
e.w.Interval = time.Hour
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
go e.w.Run(ctx)
|
||||
time.Sleep(50 * time.Millisecond) // the first run finds nothing and goes idle
|
||||
e.enqueue(t, 1)
|
||||
e.w.Wake()
|
||||
waitFor(t, "woken delivery", func() bool { return e.snd.count() == 1 })
|
||||
}
|
||||
|
||||
func TestWakeBeforeRunAndRepeatedWakesDoNotBlock(t *testing.T) {
|
||||
e := setup(t)
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
for i := 0; i < 100; i++ {
|
||||
e.w.Wake()
|
||||
}
|
||||
close(done)
|
||||
}()
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("Wake blocked")
|
||||
}
|
||||
}
|
||||
|
||||
func TestIdleRunPollsForDueRetries(t *testing.T) {
|
||||
e := setup(t)
|
||||
e.enqueue(t, 1)
|
||||
fail := int32(1)
|
||||
e.snd.fn = func(context.Context, string, []string) error {
|
||||
if atomic.LoadInt32(&fail) == 1 {
|
||||
return errors.New("down")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
e.w.Interval = 20 * time.Millisecond
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
go e.w.Run(ctx)
|
||||
waitFor(t, "first failed attempt", func() bool { st, _ := e.s.OutboxStats(); return st.Retrying == 1 })
|
||||
atomic.StoreInt32(&fail, 0)
|
||||
e.clk.set(t0.Add(2 * time.Minute)) // the backoff has passed; no Wake needed
|
||||
waitFor(t, "retry by polling", func() bool { return e.snd.count() == 1 })
|
||||
}
|
||||
|
|
@ -18,9 +18,6 @@ import (
|
|||
"codeberg.org/squidd/parsimony/internal/store"
|
||||
)
|
||||
|
||||
// bounceThreshold is the number of hard bounces after which a member is disabled.
|
||||
const bounceThreshold = 3
|
||||
|
||||
var statusRe = regexp.MustCompile(`(?mi)^Status:\s*([245])\.\d+\.\d+`)
|
||||
|
||||
func (r *Router) secret() ([]byte, error) {
|
||||
|
|
@ -170,7 +167,7 @@ func (r *Router) bounce(l store.List, tag string, data []byte) error {
|
|||
return err
|
||||
}
|
||||
if sm := statusRe.FindSubmatch(data); sm != nil && string(sm[1]) == "5" {
|
||||
_, err = r.store.RecordHardBounce(m.ID, bounceThreshold)
|
||||
_, err = r.store.RecordHardBounce(m.ID, store.BounceThreshold)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
|
|
|||
|
|
@ -41,6 +41,9 @@ const (
|
|||
ActionLeave = "leave"
|
||||
)
|
||||
|
||||
// BounceThreshold is the number of hard bounces after which a member is disabled.
|
||||
const BounceThreshold = 3
|
||||
|
||||
const (
|
||||
pendingTTL = 72 * time.Hour
|
||||
pendingRateLimit = time.Hour
|
||||
|
|
|
|||
Loading…
Reference in a new issue